00000400:00000010:2.0F:1773941269.680958:0:2291:0:(lib-cpt.c:111:cfs_cpt_table_alloc()) alloc '(cptab)': 56 at 00000000fd4f7343. 00000400:00000010:2.0:1773941269.680975:0:2291:0:(lib-cpt.c:120:cfs_cpt_table_alloc()) alloc '(cptab->ctb_nodemask)': 128 at 00000000ce00f435. 00000400:00000010:2.0:1773941269.680979:0:2291:0:(lib-cpt.c:124:cfs_cpt_table_alloc()) alloc '(cptab->ctb_cpu2cpt)': 16 at 0000000005767d5a. 00000400:00000010:2.0:1773941269.680982:0:2291:0:(lib-cpt.c:131:cfs_cpt_table_alloc()) alloc '(cptab->ctb_node2cpt)': 4 at 00000000e3bd3fd6. 00000400:00000010:2.0:1773941269.680985:0:2291:0:(lib-cpt.c:138:cfs_cpt_table_alloc()) alloc '(cptab->ctb_parts)': 64 at 000000009ee31cb8. 00000400:00000010:2.0:1773941269.680987:0:2291:0:(lib-cpt.c:150:cfs_cpt_table_alloc()) alloc '(part->cpt_nodemask)': 128 at 0000000083cc0b4f. 00000400:00000010:2.0:1773941269.680990:0:2291:0:(lib-cpt.c:154:cfs_cpt_table_alloc()) alloc '(part->cpt_distance)': 8 at 00000000f5365121. 00000400:00000010:2.0:1773941269.680992:0:2291:0:(lib-cpt.c:150:cfs_cpt_table_alloc()) alloc '(part->cpt_nodemask)': 128 at 00000000a38a51e7. 00000400:00000010:2.0:1773941269.680993:0:2291:0:(lib-cpt.c:154:cfs_cpt_table_alloc()) alloc '(part->cpt_distance)': 8 at 00000000c7eab264. 00000400:00000010:0.0F:1773941270.438838:0:2291:0:(lock.c:44:cfs_percpt_lock_create()) alloc '(pcl)': 24 at 000000007c23e96c. 00000400:00000010:0.0:1773941270.438856:0:2291:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000f03ce66e. 00000400:00000010:0.0:1773941270.438862:0:2291:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bbf22b88. 00000400:00000010:0.0:1773941270.438864:0:2291:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001634fb78. 00000400:00000010:0.0:1773941270.438866:0:2291:0:(lock.c:44:cfs_percpt_lock_create()) alloc '(pcl)': 24 at 00000000be81fa0a. 00000400:00000010:0.0:1773941270.438867:0:2291:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000af0160e7. 00000400:00000010:0.0:1773941270.438869:0:2291:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f5972138. 00000400:00000010:0.0:1773941270.438870:0:2291:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d3b433cd. 00000400:00000200:0.0:1773941270.438999:0:2291:0:(api-ni.c:982:lnet_register_lnd()) lo LND registered 00000400:00000200:2.0:1773941270.494253:0:2307:0:(api-ni.c:982:lnet_register_lnd()) tcp LND registered 00000020:02000000:3.0F:1773941270.680664:0:2320:0:(class_obd.c:784:obdclass_init()) Lustre: Build Version: 2.17.50_193_gf5bca74 00000020:00000010:3.0:1773941270.683454:0:2320:0:(lustre_handles.c:148:class_handle_init()) kmalloced '(handle_hash)': 2097152 at 00000000ed14d888. 00000001:00000010:3.0:1773941270.684251:0:2320:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 00000000d9d2d749. 00000001:00000010:3.0:1773941270.684257:0:2320:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 00000000fe4165f7. 00000001:00000010:3.0:1773941270.684260:0:2320:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000001277c87a. 00000001:00000010:3.0:1773941270.684263:0:2320:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000e4c32da0. 00000001:00000010:3.0:1773941270.684266:0:2320:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000001bec9738. 00000001:00000010:3.0:1773941270.684268:0:2320:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000f332b2a4. 00000020:00000010:3.0:1773941270.684308:0:2320:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ffe5ade3. 00000020:00000010:3.0:1773941270.684312:0:2320:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000b6cfc6d1. 00000020:00000010:3.0:1773941270.684315:0:2320:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000dea3a9bd. 00000020:00000010:3.0:1773941270.684318:0:2320:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000b95af440. 00000001:00000010:3.0:1773941270.684321:0:2320:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 0000000053aba3f2. 00000020:00000010:3.0:1773941270.684355:0:2320:0:(cl_object.c:1176:cl_global_init()) kmalloced '(cl_envs)': 192 at 000000000e9d649a. 00000020:00000010:3.0:1773941270.684499:0:2320:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000042d97e40. 00000020:00000010:3.0:1773941270.684503:0:2320:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000a6634210. 00000020:00000010:3.0:1773941270.684505:0:2320:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 000000001059bb46. 00000020:00000010:3.0:1773941270.684508:0:2320:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000003e4b3d6a. 00000020:00000010:3.0:1773941270.684511:0:2320:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ec749212. 00000020:00000010:3.0:1773941270.684513:0:2320:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000088b71901. 00000020:00000010:3.0:1773941270.684515:0:2320:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000f98fd12c. 00000020:00000010:3.0:1773941270.684518:0:2320:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005dd1139f. 00000020:00000010:3.0:1773941270.684520:0:2320:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e4038bb9. 00000020:00000010:3.0:1773941270.684522:0:2320:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000009aa51500. 00000020:00000010:3.0:1773941270.684523:0:2320:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000032b7b1e. 00000020:00000010:3.0:1773941270.684526:0:2320:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000060fd190c. 00000020:00000010:3.0:1773941270.684528:0:2320:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000033851260. 00000020:00000010:3.0:1773941270.684530:0:2320:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000affc4fdc. 00000020:00000010:3.0:1773941270.684533:0:2320:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 000000007386b20f. 00000020:00000010:3.0:1773941270.684535:0:2320:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000020640975. 02000000:00000010:3.0:1773941270.684549:0:2320:0:(page_pools.c:1061:obd_pool_init()) kmalloced '(page_pools)': 112 at 000000001aed8892. 02000000:00000010:3.0:1773941270.684551:0:2320:0:(page_pools.c:1065:obd_pool_init()) kmalloced '(pool_shrinkers)': 112 at 00000000e83c9164. 02000000:00000010:3.0:1773941270.684560:0:2320:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000875b46a9. 02000000:00000010:3.0:1773941270.684566:0:2320:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 1056 at 000000007f049967. 00000001:00000010:3.0:1773941270.684569:0:2320:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 0000000072605001. 00000020:00000010:3.0:1773941270.684591:0:2320:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 000000007d1a12b8. 00000020:00000010:3.0:1773941270.684598:0:2320:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 000000007d1a12b8. 02000000:00000010:3.0:1773941270.684601:0:2320:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000803b9477. 02000000:00000010:3.0:1773941270.684603:0:2320:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 528 at 00000000713efee4. 00000001:00000010:3.0:1773941270.684605:0:2320:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 000000002c9669ea. 00000020:00000010:3.0:1773941270.684621:0:2320:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 000000007d1a12b8. 00000020:00000010:3.0:1773941270.684627:0:2320:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 000000007d1a12b8. 02000000:00000010:3.0:1773941270.684630:0:2320:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000605355cd. 02000000:00000010:3.0:1773941270.684632:0:2320:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 264 at 00000000f4783a4d. 00000001:00000010:3.0:1773941270.684634:0:2320:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 0000000007e953d3. 00000020:00000010:3.0:1773941270.684648:0:2320:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 000000007d1a12b8. 00000020:00000010:3.0:1773941270.684670:0:2320:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 000000007d1a12b8. 02000000:00000010:3.0:1773941270.684674:0:2320:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000f9d0bcd1. 02000000:00000010:3.0:1773941270.684676:0:2320:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 136 at 000000007d1a12b8. 00000001:00000010:3.0:1773941270.684678:0:2320:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 0000000058f6a9f2. 00000020:00000010:3.0:1773941270.684693:0:2320:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000c895abb2. 00000020:00000010:3.0:1773941270.684699:0:2320:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000c895abb2. 02000000:00000010:3.0:1773941270.684702:0:2320:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000d198de45. 02000000:00000010:3.0:1773941270.684730:0:2320:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 72 at 000000007db0be2b. 00000001:00000010:3.0:1773941270.684733:0:2320:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 000000007f7b03be. 00000020:00000010:3.0:1773941270.684763:0:2320:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000c895abb2. 00000020:00000010:3.0:1773941270.684769:0:2320:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000c895abb2. 02000000:00000010:3.0:1773941270.684772:0:2320:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000625679ec. 02000000:00000010:3.0:1773941270.684774:0:2320:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 40 at 00000000bd862c06. 00000001:00000010:3.0:1773941270.684776:0:2320:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 0000000063870fe0. 00000020:00000010:3.0:1773941270.684791:0:2320:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000c895abb2. 00000020:00000010:3.0:1773941270.684797:0:2320:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000c895abb2. 02000000:00000010:3.0:1773941270.684800:0:2320:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 0000000044b4ae14. 02000000:00000010:3.0:1773941270.684802:0:2320:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 24 at 0000000018c7c5c7. 00000001:00000010:3.0:1773941270.684807:0:2320:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000fd4cefc0. 00000020:00000010:3.0:1773941270.684823:0:2320:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000c895abb2. 00000020:00000010:3.0:1773941270.684829:0:2320:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000c895abb2. 02000000:00000010:3.0:1773941270.684832:0:2320:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000e41a5c9b. 02000000:00000010:3.0:1773941270.684834:0:2320:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 16 at 0000000091c4715d. 00000001:00000010:3.0:1773941270.684836:0:2320:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 0000000056e2acac. 00000020:00000010:3.0:1773941270.684849:0:2320:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000c895abb2. 00000020:00000010:3.0:1773941270.684869:0:2320:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000c895abb2. 02000000:00000010:3.0:1773941270.684873:0:2320:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 0000000074f4b7f6. 02000000:00000010:3.0:1773941270.684875:0:2320:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 000000005458455f. 00000001:00000010:3.0:1773941270.684877:0:2320:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 000000006ae91e1d. 00000020:00000010:3.0:1773941270.684892:0:2320:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000c895abb2. 00000020:00000010:3.0:1773941270.684898:0:2320:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000c895abb2. 02000000:00000010:3.0:1773941270.684901:0:2320:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 000000003b0616c7. 02000000:00000010:3.0:1773941270.684903:0:2320:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 0000000071d4d054. 00000001:00000010:3.0:1773941270.684904:0:2320:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000b4f55b04. 00000020:00000010:3.0:1773941270.684919:0:2320:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000c895abb2. 00000020:00000010:3.0:1773941270.684925:0:2320:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000c895abb2. 02000000:00000010:3.0:1773941270.684928:0:2320:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 000000008008c129. 02000000:00000010:3.0:1773941270.684930:0:2320:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 00000000c0af2c06. 00000001:00000010:3.0:1773941270.684932:0:2320:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000b970af36. 00000020:00000010:3.0:1773941270.684946:0:2320:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000c895abb2. 00000020:00000010:3.0:1773941270.684951:0:2320:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000c895abb2. 02000000:00000010:3.0:1773941270.684954:0:2320:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 000000001b0eb646. 02000000:00000010:3.0:1773941270.684956:0:2320:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 00000000571ee82d. 00000001:00000010:3.0:1773941270.684958:0:2320:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000d141cd6c. 00000020:00000010:3.0:1773941270.684981:0:2320:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 0000000026c8a636. 00000020:00000010:3.0:1773941270.684987:0:2320:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 0000000026c8a636. 02000000:00000010:3.0:1773941270.684992:0:2320:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 0000000065633668. 02000000:00000010:3.0:1773941270.684994:0:2320:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 00000000d381237f. 00000001:00000010:3.0:1773941270.684996:0:2320:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000434dc05b. 00000020:00000010:3.0:1773941270.685013:0:2320:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 0000000026c8a636. 00000020:00000010:3.0:1773941270.685032:0:2320:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 0000000026c8a636. 02000000:00000010:3.0:1773941270.685035:0:2320:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000cfacddda. 02000000:00000010:3.0:1773941270.685037:0:2320:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 000000001538e3b3. 00000001:00000010:3.0:1773941270.685039:0:2320:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000fac9e256. 00000020:00000010:3.0:1773941270.685054:0:2320:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 0000000026c8a636. 00000020:00000010:3.0:1773941270.685059:0:2320:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 0000000026c8a636. 00000400:00000010:1.0F:1773941270.958571:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000e01a0620. 00000400:00000010:1.0:1773941270.958585:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c5b98bf5. 00000400:00000010:1.0:1773941270.958586:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004f6df6c3. 00000100:00000010:1.0:1773941270.958590:0:2335:0:(service.c:3571:ptlrpc_hr_init()) kmalloced '(hrp->hrp_thrs)': 96 at 00000000b9a55dcc. 00000100:00000010:1.0:1773941270.958592:0:2335:0:(service.c:3571:ptlrpc_hr_init()) kmalloced '(hrp->hrp_thrs)': 96 at 00000000dfe9d377. 00000100:00000010:2.0:1773941270.958923:0:2337:0:(service.c:3214:ptlrpc_hr_main()) kmalloced '(env)': 56 at 000000007fab3c2d. 00000020:00000010:0.0:1773941270.958988:0:2337:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006f719f63. 00000020:00000010:0.0:1773941270.958994:0:2337:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000008d6cdf90. 00000040:00000010:0.0:1773941270.958998:0:2337:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000074d2915e. 00000020:00000010:0.0:1773941270.959002:0:2337:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000f731321e. 00000020:00000010:0.0:1773941270.959007:0:2337:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 0000000033b90143. 00000100:00000010:2.0:1773941270.959196:0:2338:0:(service.c:3214:ptlrpc_hr_main()) kmalloced '(env)': 56 at 00000000bb26c992. 00000020:00000010:2.0:1773941270.959205:0:2338:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005d8bfc3f. 00000020:00000010:2.0:1773941270.959208:0:2338:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000007a426802. 00000040:00000010:2.0:1773941270.959211:0:2338:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000f91b1f22. 00000020:00000010:2.0:1773941270.959214:0:2338:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000da03aa02. 00000020:00000010:2.0:1773941270.959216:0:2338:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000001f4669e7. 00000100:00000200:1.0:1773941270.959257:0:2335:0:(events.c:597:ptlrpc_ni_init()) My pid is: 3039 00000400:00000010:1.0:1773941270.959428:0:2335:0:(api-ni.c:679:lnet_create_remote_nets_table()) alloc '(hash)': 1024 at 00000000b2082016. 00000400:00000010:1.0:1773941270.959431:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000cec08797. 00000400:00000010:1.0:1773941270.959434:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000777ad678. 00000400:00000010:1.0:1773941270.959435:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000e1c70996. 00000400:00000010:1.0:1773941270.959438:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000cd25ed3c. 00000400:00000010:1.0:1773941270.959439:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000970c13fc. 00000400:00000010:1.0:1773941270.959441:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 0000000001dd7ed1. 00000400:00000010:1.0:1773941270.959446:0:2335:0:(peer.c:108:lnet_peer_tables_create()) alloc '(hash)': 8192 at 0000000093f2b42b. 00000400:00000010:1.0:1773941270.959458:0:2335:0:(peer.c:108:lnet_peer_tables_create()) alloc '(hash)': 8192 at 000000001fd99bcb. 00000400:00000010:1.0:1773941270.959464:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000094a0373f. 00000400:00000010:1.0:1773941270.959466:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 000000000dde1ed1. 00000400:00000010:1.0:1773941270.959467:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000f0c23e44. 00000400:00000010:1.0:1773941270.959471:0:2335:0:(lib-msg.c:1260:lnet_msg_container_setup()) alloc '(container->msc_finalizers)': 16 at 0000000086ab5761. 00000400:00000010:1.0:1773941270.959474:0:2335:0:(lib-msg.c:1270:lnet_msg_container_setup()) alloc '(container->msc_resenders)': 16 at 0000000006fa7ef2. 00000400:00000010:1.0:1773941270.959476:0:2335:0:(lib-msg.c:1260:lnet_msg_container_setup()) alloc '(container->msc_finalizers)': 16 at 0000000077631926. 00000400:00000010:1.0:1773941270.959478:0:2335:0:(lib-msg.c:1270:lnet_msg_container_setup()) alloc '(container->msc_resenders)': 16 at 00000000158680bc. 00000400:00000010:1.0:1773941270.959529:0:2335:0:(api-ni.c:1174:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 0000000088102369. 00000400:00000010:1.0:1773941270.959544:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000ddb69d66. 00000400:00000010:1.0:1773941270.959545:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000027b0bd9a. 00000400:00000010:1.0:1773941270.959546:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b8014cea. 00000400:00000010:1.0:1773941270.959589:0:2335:0:(api-ni.c:1174:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 00000000d361b4d5. 00000400:00000010:1.0:1773941270.959660:0:2335:0:(api-ni.c:1174:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 0000000012efe973. 00000400:00000010:1.0:1773941270.959679:0:2335:0:(lib-ptl.c:867:lnet_portals_create()) alloc '(the_lnet.ln_portals)': 512 at 0000000071634a36. 00000400:00000010:1.0:1773941270.959690:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000016c67713. 00000400:00000010:1.0:1773941270.959693:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000002a4b33e. 00000400:00000010:1.0:1773941270.959696:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008c3c7971. 00000400:00000010:1.0:1773941270.959699:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f451af2b. 00000400:00000010:1.0:1773941270.959735:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005bc1df27. 00000400:00000010:1.0:1773941270.959741:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ba9cdfb4. 00000400:00000010:1.0:1773941270.959745:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000b9d89dd0. 00000400:00000010:1.0:1773941270.959748:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000093316d2e. 00000400:00000010:1.0:1773941270.959750:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000087ee8b4b. 00000400:00000010:1.0:1773941270.959752:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f8c638b1. 00000400:00000010:1.0:1773941270.959756:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003c5d6268. 00000400:00000010:1.0:1773941270.959768:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002b9860a5. 00000400:00000010:1.0:1773941270.959772:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000ead64a99. 00000400:00000010:1.0:1773941270.959774:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000c9343622. 00000400:00000010:1.0:1773941270.959776:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002d85968e. 00000400:00000010:1.0:1773941270.959778:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007a01ecf1. 00000400:00000010:1.0:1773941270.959782:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007acc643c. 00000400:00000010:1.0:1773941270.959786:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000974d8d48. 00000400:00000010:1.0:1773941270.959789:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000043d79f4d. 00000400:00000010:1.0:1773941270.959790:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000080a8dd76. 00000400:00000010:1.0:1773941270.959792:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004bb9de03. 00000400:00000010:1.0:1773941270.959793:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000034688b85. 00000400:00000010:1.0:1773941270.959797:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000b73e15a. 00000400:00000010:1.0:1773941270.959806:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000fbde3f05. 00000400:00000010:1.0:1773941270.959810:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000019436eb2. 00000400:00000010:1.0:1773941270.959812:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000053de7c1b. 00000400:00000010:1.0:1773941270.959813:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fe379c49. 00000400:00000010:1.0:1773941270.959816:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002661d76c. 00000400:00000010:1.0:1773941270.959819:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000094a33987. 00000400:00000010:1.0:1773941270.959823:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c42cbd3f. 00000400:00000010:1.0:1773941270.959825:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000fa4833ea. 00000400:00000010:1.0:1773941270.959827:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000004bdf7aea. 00000400:00000010:1.0:1773941270.959830:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000071149fbd. 00000400:00000010:1.0:1773941270.959832:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000207715c3. 00000400:00000010:1.0:1773941270.959836:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002133dede. 00000400:00000010:1.0:1773941270.959847:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008743a542. 00000400:00000010:1.0:1773941270.959851:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000d3f96a7d. 00000400:00000010:1.0:1773941270.959858:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000db09127f. 00000400:00000010:1.0:1773941270.959860:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d742f654. 00000400:00000010:1.0:1773941270.959862:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007af0d134. 00000400:00000010:1.0:1773941270.959868:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ca6cf166. 00000400:00000010:1.0:1773941270.959875:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a84f81ad. 00000400:00000010:1.0:1773941270.959877:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000059a10312. 00000400:00000010:1.0:1773941270.959879:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000078d05e20. 00000400:00000010:1.0:1773941270.959881:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008e3dbcb4. 00000400:00000010:1.0:1773941270.959883:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000039849c3e. 00000400:00000010:1.0:1773941270.959887:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000085b93457. 00000400:00000010:1.0:1773941270.959897:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b4e6123a. 00000400:00000010:1.0:1773941270.959901:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000fb363497. 00000400:00000010:1.0:1773941270.959903:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000009106148b. 00000400:00000010:1.0:1773941270.959905:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000057bc71a. 00000400:00000010:1.0:1773941270.959907:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f2dc0ffe. 00000400:00000010:1.0:1773941270.959912:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000026b2a7b6. 00000400:00000010:1.0:1773941270.959917:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000144ab37c. 00000400:00000010:1.0:1773941270.959919:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000d09e27a5. 00000400:00000010:1.0:1773941270.959921:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000001e757f29. 00000400:00000010:1.0:1773941270.959923:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000059e28d94. 00000400:00000010:1.0:1773941270.959937:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d23a8dde. 00000400:00000010:1.0:1773941270.959943:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d2c990c4. 00000400:00000010:1.0:1773941270.959952:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000861e94ae. 00000400:00000010:1.0:1773941270.959955:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000000356e48d. 00000400:00000010:1.0:1773941270.959958:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000a4c85111. 00000400:00000010:1.0:1773941270.959960:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000666d667f. 00000400:00000010:1.0:1773941270.959961:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000015cc8e5b. 00000400:00000010:1.0:1773941270.959965:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d60fd47b. 00000400:00000010:1.0:1773941270.959969:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000eb4c49fc. 00000400:00000010:1.0:1773941270.959971:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000f0935e11. 00000400:00000010:1.0:1773941270.959972:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000007fb32133. 00000400:00000010:1.0:1773941270.959974:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000799e045. 00000400:00000010:1.0:1773941270.959976:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002ea9a28a. 00000400:00000010:1.0:1773941270.959983:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000411b9571. 00000400:00000010:1.0:1773941270.959992:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000fe6dad3c. 00000400:00000010:1.0:1773941270.959995:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000004316bd0f. 00000400:00000010:1.0:1773941270.959997:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000c488ddf2. 00000400:00000010:1.0:1773941270.959999:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f70b7b69. 00000400:00000010:1.0:1773941270.960000:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000014797b0b. 00000400:00000010:1.0:1773941270.960004:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000086896856. 00000400:00000010:1.0:1773941270.960009:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a269e995. 00000400:00000010:1.0:1773941270.960011:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000008c71ac5d. 00000400:00000010:1.0:1773941270.960013:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000061711b78. 00000400:00000010:1.0:1773941270.960014:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f77e05ca. 00000400:00000010:1.0:1773941270.960016:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005192c96e. 00000400:00000010:1.0:1773941270.960020:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003de238a8. 00000400:00000010:1.0:1773941270.960028:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000450ba9e5. 00000400:00000010:1.0:1773941270.960031:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000331445d7. 00000400:00000010:1.0:1773941270.960033:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000f759e0d0. 00000400:00000010:1.0:1773941270.960034:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000034b3210f. 00000400:00000010:1.0:1773941270.960036:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000071d259e1. 00000400:00000010:1.0:1773941270.960039:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002cfadcc5. 00000400:00000010:1.0:1773941270.960043:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c7d54034. 00000400:00000010:1.0:1773941270.960045:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000006060e0d6. 00000400:00000010:1.0:1773941270.960047:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000026061fb4. 00000400:00000010:1.0:1773941270.960048:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d6577719. 00000400:00000010:1.0:1773941270.960050:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a9efe591. 00000400:00000010:1.0:1773941270.960053:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000746c0f70. 00000400:00000010:1.0:1773941270.960061:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c67b7014. 00000400:00000010:1.0:1773941270.960064:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000e0bb0b59. 00000400:00000010:1.0:1773941270.960066:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000080c12b0. 00000400:00000010:1.0:1773941270.960067:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dade4ae0. 00000400:00000010:1.0:1773941270.960069:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f5b882a5. 00000400:00000010:1.0:1773941270.960072:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000097bd3ddd. 00000400:00000010:1.0:1773941270.960077:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009abf2196. 00000400:00000010:1.0:1773941270.960079:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000008dbd860c. 00000400:00000010:1.0:1773941270.960085:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000004480b69d. 00000400:00000010:1.0:1773941270.960087:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000878b99a5. 00000400:00000010:1.0:1773941270.960089:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d44ae0c4. 00000400:00000010:1.0:1773941270.960092:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000dec1f59b. 00000400:00000010:1.0:1773941270.960101:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003ba62381. 00000400:00000010:1.0:1773941270.960114:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000008eb64131. 00000400:00000010:1.0:1773941270.960116:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000000f7484f9. 00000400:00000010:1.0:1773941270.960119:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000aa45536b. 00000400:00000010:1.0:1773941270.960120:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000611aaf7e. 00000400:00000010:1.0:1773941270.960125:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006ee0dbb6. 00000400:00000010:1.0:1773941270.960130:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f0cc199c. 00000400:00000010:1.0:1773941270.960132:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000df50fb27. 00000400:00000010:1.0:1773941270.960134:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000041b5e70c. 00000400:00000010:1.0:1773941270.960136:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001dc6991c. 00000400:00000010:1.0:1773941270.960138:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dfd6827f. 00000400:00000010:1.0:1773941270.960142:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000004021b7d. 00000400:00000010:1.0:1773941270.960153:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006b88cf9b. 00000400:00000010:1.0:1773941270.960157:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000006d32180d. 00000400:00000010:1.0:1773941270.960159:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000794fefa4. 00000400:00000010:1.0:1773941270.960162:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000007781395. 00000400:00000010:1.0:1773941270.960164:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b3250a7f. 00000400:00000010:1.0:1773941270.960167:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c6cd7988. 00000400:00000010:1.0:1773941270.960172:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000614742fb. 00000400:00000010:1.0:1773941270.960175:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000000af6a2b6. 00000400:00000010:1.0:1773941270.960177:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000c2054d52. 00000400:00000010:1.0:1773941270.960179:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000697498f8. 00000400:00000010:1.0:1773941270.960181:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006e9208a3. 00000400:00000010:1.0:1773941270.960185:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007a71732a. 00000400:00000010:1.0:1773941270.960194:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003141a45d. 00000400:00000010:1.0:1773941270.960198:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000bbd8e159. 00000400:00000010:1.0:1773941270.960200:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000006999b9d2. 00000400:00000010:1.0:1773941270.960202:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c43d5dd3. 00000400:00000010:1.0:1773941270.960205:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000419d89fa. 00000400:00000010:1.0:1773941270.960213:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000006c5e72b. 00000400:00000010:1.0:1773941270.960217:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000929ec4fb. 00000400:00000010:1.0:1773941270.960220:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000085f7589e. 00000400:00000010:1.0:1773941270.960222:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000274a48ed. 00000400:00000010:1.0:1773941270.960225:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ae01af5a. 00000400:00000010:1.0:1773941270.960227:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000077b1ec83. 00000400:00000010:1.0:1773941270.960230:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000fd886b72. 00000400:00000010:1.0:1773941270.960240:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e8b4c863. 00000400:00000010:1.0:1773941270.960244:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000c327c9ba. 00000400:00000010:1.0:1773941270.960246:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000474c959e. 00000400:00000010:1.0:1773941270.960248:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e374e054. 00000400:00000010:1.0:1773941270.960250:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f17e6622. 00000400:00000010:1.0:1773941270.960254:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006389f5bb. 00000400:00000010:1.0:1773941270.960258:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000984208f4. 00000400:00000010:1.0:1773941270.960262:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000005cbc28d0. 00000400:00000010:1.0:1773941270.960264:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000013e52e49. 00000400:00000010:1.0:1773941270.960266:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003473842d. 00000400:00000010:1.0:1773941270.960267:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000009f24a94. 00000400:00000010:1.0:1773941270.960271:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000c13bf1e. 00000400:00000010:1.0:1773941270.960281:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009b745941. 00000400:00000010:1.0:1773941270.960284:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000053a31b36. 00000400:00000010:1.0:1773941270.960286:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000447ec765. 00000400:00000010:1.0:1773941270.960288:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000960bf9dc. 00000400:00000010:1.0:1773941270.960290:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000c7c8a16. 00000400:00000010:1.0:1773941270.960294:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001d9eadec. 00000400:00000010:1.0:1773941270.960298:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000056a68365. 00000400:00000010:1.0:1773941270.960300:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000c9edb36b. 00000400:00000010:1.0:1773941270.960302:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000b4062ef3. 00000400:00000010:1.0:1773941270.960304:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000080761c1b. 00000400:00000010:1.0:1773941270.960307:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009175943a. 00000400:00000010:1.0:1773941270.960310:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f04692ea. 00000400:00000010:1.0:1773941270.960321:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b6119cbb. 00000400:00000010:1.0:1773941270.960325:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000836f353f. 00000400:00000010:1.0:1773941270.960331:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000004d993eb6. 00000400:00000010:1.0:1773941270.960333:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ba66adf7. 00000400:00000010:1.0:1773941270.960335:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000770d4190. 00000400:00000010:1.0:1773941270.960339:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001c1863a7. 00000400:00000010:1.0:1773941270.960343:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000039706f81. 00000400:00000010:1.0:1773941270.960346:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000520500b4. 00000400:00000010:1.0:1773941270.960348:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000501d2309. 00000400:00000010:1.0:1773941270.960349:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000654b6e59. 00000400:00000010:1.0:1773941270.960351:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000edf750da. 00000400:00000010:1.0:1773941270.960354:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e8c2b509. 00000400:00000010:1.0:1773941270.960365:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000068d56efe. 00000400:00000010:1.0:1773941270.960369:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000003582b507. 00000400:00000010:1.0:1773941270.960371:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000066a5eb88. 00000400:00000010:1.0:1773941270.960373:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dec74145. 00000400:00000010:1.0:1773941270.960374:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007a83ef51. 00000400:00000010:1.0:1773941270.960378:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000048f85d00. 00000400:00000010:1.0:1773941270.960383:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000031270323. 00000400:00000010:1.0:1773941270.960385:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000df0273f1. 00000400:00000010:1.0:1773941270.960387:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000b88b4580. 00000400:00000010:1.0:1773941270.960389:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ee33370d. 00000400:00000010:1.0:1773941270.960390:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000038d8ca59. 00000400:00000010:1.0:1773941270.960394:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d438e2e2. 00000400:00000010:1.0:1773941270.960405:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000048002e9c. 00000400:00000010:1.0:1773941270.960408:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000017e5b981. 00000400:00000010:1.0:1773941270.960410:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000a2db58cf. 00000400:00000010:1.0:1773941270.960412:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000092511b5d. 00000400:00000010:1.0:1773941270.960413:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c880b9c7. 00000400:00000010:1.0:1773941270.960417:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000157213d3. 00000400:00000010:1.0:1773941270.960421:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000649a82c0. 00000400:00000010:1.0:1773941270.960423:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000080012ecf. 00000400:00000010:1.0:1773941270.960424:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000067f850f0. 00000400:00000010:1.0:1773941270.960426:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dde97509. 00000400:00000010:1.0:1773941270.960427:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000c3ee41f. 00000400:00000010:1.0:1773941270.960435:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003689233a. 00000400:00000010:1.0:1773941270.960442:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000cad9a82d. 00000400:00000010:1.0:1773941270.960444:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000007787528e. 00000400:00000010:1.0:1773941270.960445:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000a7d3c5d1. 00000400:00000010:1.0:1773941270.960446:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004013c6ca. 00000400:00000010:1.0:1773941270.960447:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000052b23923. 00000400:00000010:1.0:1773941270.960449:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000460bcc7. 00000400:00000010:1.0:1773941270.960452:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000fcbb0d1d. 00000400:00000010:1.0:1773941270.960453:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000e11314c3. 00000400:00000010:1.0:1773941270.960454:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000006e2fb0fe. 00000400:00000010:1.0:1773941270.960455:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d7f005b9. 00000400:00000010:1.0:1773941270.960456:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f6757f31. 00000400:00000010:1.0:1773941270.960458:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000bf76f392. 00000400:00000010:1.0:1773941270.960465:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000040e2d3fc. 00000400:00000010:1.0:1773941270.960467:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000930fd111. 00000400:00000010:1.0:1773941270.960468:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000a91803c3. 00000400:00000010:1.0:1773941270.960469:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a23f70b3. 00000400:00000010:1.0:1773941270.960470:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000058a54444. 00000400:00000010:1.0:1773941270.960473:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000701d79ef. 00000400:00000010:1.0:1773941270.960477:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000581695c0. 00000400:00000010:1.0:1773941270.960478:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000017356ff7. 00000400:00000010:1.0:1773941270.960479:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000fb186e74. 00000400:00000010:1.0:1773941270.960480:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f4d8a5af. 00000400:00000010:1.0:1773941270.960481:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004bc6fa8b. 00000400:00000010:1.0:1773941270.960484:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009c80e99c. 00000400:00000010:1.0:1773941270.960491:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001f0a3819. 00000400:00000010:1.0:1773941270.960493:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000086feda1c. 00000400:00000010:1.0:1773941270.960494:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000096aa314a. 00000400:00000010:1.0:1773941270.960495:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fdf76267. 00000400:00000010:1.0:1773941270.960495:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000030691283. 00000400:00000010:1.0:1773941270.960499:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000008ddcf90. 00000400:00000010:1.0:1773941270.960502:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ffa9b08b. 00000400:00000010:1.0:1773941270.960503:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000add463d6. 00000400:00000010:1.0:1773941270.960508:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000e5b6ceff. 00000400:00000010:1.0:1773941270.960509:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000eab366c3. 00000400:00000010:1.0:1773941270.960510:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000282a023b. 00000400:00000010:1.0:1773941270.960513:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d2f9244d. 00000400:00000010:1.0:1773941270.960519:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000deb4dbce. 00000400:00000010:1.0:1773941270.960521:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000a7266697. 00000400:00000010:1.0:1773941270.960522:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000000fd8ae04. 00000400:00000010:1.0:1773941270.960523:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c207ef54. 00000400:00000010:1.0:1773941270.960524:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000027b3c930. 00000400:00000010:1.0:1773941270.960526:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006ebc535d. 00000400:00000010:1.0:1773941270.960529:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005a59eaf8. 00000400:00000010:1.0:1773941270.960530:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000005216c2b6. 00000400:00000010:1.0:1773941270.960531:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000db9074c7. 00000400:00000010:1.0:1773941270.960532:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ec20fd47. 00000400:00000010:1.0:1773941270.960541:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008e2cdc9f. 00000400:00000010:1.0:1773941270.960544:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000db3068da. 00000400:00000010:1.0:1773941270.960549:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000047f8e290. 00000400:00000010:1.0:1773941270.960557:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000e5a8aacc. 00000400:00000010:1.0:1773941270.960558:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000fa7c262a. 00000400:00000010:1.0:1773941270.960559:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000075eb676d. 00000400:00000010:1.0:1773941270.960560:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000035274392. 00000400:00000010:1.0:1773941270.960563:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000075cd9ffa. 00000400:00000010:1.0:1773941270.960565:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000233fe92b. 00000400:00000010:1.0:1773941270.960567:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000f1701240. 00000400:00000010:1.0:1773941270.960568:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000006c3193ec. 00000400:00000010:1.0:1773941270.960569:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002b99452a. 00000400:00000010:1.0:1773941270.960569:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000553f20f0. 00000400:00000010:1.0:1773941270.960572:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000cc72ede. 00000400:00000010:1.0:1773941270.960578:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000dcda96af. 00000400:00000010:1.0:1773941270.960580:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000000614c4cd. 00000400:00000010:1.0:1773941270.960581:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000f886d4ce. 00000400:00000010:1.0:1773941270.960582:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000032283088. 00000400:00000010:1.0:1773941270.960583:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c05f1c62. 00000400:00000010:1.0:1773941270.960588:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f3369260. 00000400:00000010:1.0:1773941270.960591:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000074cc0ad5. 00000400:00000010:1.0:1773941270.960592:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000223b7312. 00000400:00000010:1.0:1773941270.960593:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000291a4603. 00000400:00000010:1.0:1773941270.960594:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009d590a24. 00000400:00000010:1.0:1773941270.960595:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007ef0ad67. 00000400:00000010:1.0:1773941270.960597:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001c13759d. 00000400:00000010:1.0:1773941270.960604:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e82d1627. 00000400:00000010:1.0:1773941270.960605:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000007393cc9e. 00000400:00000010:1.0:1773941270.960607:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000080b98ff6. 00000400:00000010:1.0:1773941270.960608:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000079f25984. 00000400:00000010:1.0:1773941270.960609:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000034d9bb64. 00000400:00000010:1.0:1773941270.960611:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a3cdff4d. 00000400:00000010:1.0:1773941270.960614:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000cfa475d. 00000400:00000010:1.0:1773941270.960615:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000435bc152. 00000400:00000010:1.0:1773941270.960616:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000334b51fe. 00000400:00000010:1.0:1773941270.960617:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d208327d. 00000400:00000010:1.0:1773941270.960618:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007c9625a6. 00000400:00000010:1.0:1773941270.960620:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000045296fa2. 00000400:00000010:1.0:1773941270.960626:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000028e4564. 00000400:00000010:1.0:1773941270.960628:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000004bcabb94. 00000400:00000010:1.0:1773941270.960629:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000e5b721ce. 00000400:00000010:1.0:1773941270.960630:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000067eaf554. 00000400:00000010:1.0:1773941270.960631:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fba06f0b. 00000400:00000010:1.0:1773941270.960634:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000ee54a1d. 00000400:00000010:1.0:1773941270.960636:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000dd1beb7f. 00000400:00000010:1.0:1773941270.960638:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000810742fd. 00000400:00000010:1.0:1773941270.960639:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000bd04bb1f. 00000400:00000010:1.0:1773941270.960640:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000eb4bf933. 00000400:00000010:1.0:1773941270.960640:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000047f213a4. 00000400:00000010:1.0:1773941270.960643:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006c1a6bb0. 00000400:00000010:1.0:1773941270.960650:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b418ed23. 00000400:00000010:1.0:1773941270.960652:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000efab4557. 00000400:00000010:1.0:1773941270.960655:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000281f50e4. 00000400:00000010:1.0:1773941270.960656:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000035c09576. 00000400:00000010:1.0:1773941270.960657:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000bc09ad94. 00000400:00000010:1.0:1773941270.960661:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000966283a7. 00000400:00000010:1.0:1773941270.960665:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000467f4d44. 00000400:00000010:1.0:1773941270.960666:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000008eb6844a. 00000400:00000010:1.0:1773941270.960667:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000eb0fd803. 00000400:00000010:1.0:1773941270.960668:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c545175a. 00000400:00000010:1.0:1773941270.960668:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000150b0d32. 00000400:00000010:1.0:1773941270.960672:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c8f7f71c. 00000400:00000010:1.0:1773941270.960678:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004ddac525. 00000400:00000010:1.0:1773941270.960679:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000fb643d94. 00000400:00000010:1.0:1773941270.960680:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000d738fff5. 00000400:00000010:1.0:1773941270.960681:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c4ff95be. 00000400:00000010:1.0:1773941270.960682:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007c74ac4c. 00000400:00000010:1.0:1773941270.960684:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009b54bb34. 00000400:00000010:1.0:1773941270.960687:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d6daf8b4. 00000400:00000010:1.0:1773941270.960688:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000008405783c. 00000400:00000010:1.0:1773941270.960689:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000863b170e. 00000400:00000010:1.0:1773941270.960690:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c50fe4b6. 00000400:00000010:1.0:1773941270.960691:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f46ac40a. 00000400:00000010:1.0:1773941270.960693:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000000fb348ba. 00000400:00000010:1.0:1773941270.960700:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e23fc3fa. 00000400:00000010:1.0:1773941270.960715:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000003254d91a. 00000400:00000010:1.0:1773941270.960717:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000c93c9625. 00000400:00000010:1.0:1773941270.960719:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005f935cfa. 00000400:00000010:1.0:1773941270.960720:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c001bcf5. 00000400:00000010:1.0:1773941270.960724:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f431b608. 00000400:00000010:1.0:1773941270.960728:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ef0cb7e0. 00000400:00000010:1.0:1773941270.960730:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000093ecf63e. 00000400:00000010:1.0:1773941270.960731:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000b21cf97b. 00000400:00000010:1.0:1773941270.960732:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000072177393. 00000400:00000010:1.0:1773941270.960733:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c7effe0a. 00000400:00000010:1.0:1773941270.960740:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d9cfd1e2. 00000400:00000010:1.0:1773941270.960748:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000049670496. 00000400:00000010:1.0:1773941270.960750:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000002a143d96. 00000400:00000010:1.0:1773941270.960751:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000af052d89. 00000400:00000010:1.0:1773941270.960752:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002a39225a. 00000400:00000010:1.0:1773941270.960754:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000962e7139. 00000400:00000010:1.0:1773941270.960757:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000083e3638b. 00000400:00000010:1.0:1773941270.960760:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005a5df202. 00000400:00000010:1.0:1773941270.960761:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000050834e12. 00000400:00000010:1.0:1773941270.960762:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000001b2e18b7. 00000400:00000010:1.0:1773941270.960763:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000054adcdc8. 00000400:00000010:1.0:1773941270.960764:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000788fbf85. 00000400:00000010:1.0:1773941270.960767:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008aedec60. 00000400:00000010:1.0:1773941270.960774:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000081a46d87. 00000400:00000010:1.0:1773941270.960776:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000068ba2dd5. 00000400:00000010:1.0:1773941270.960777:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000feffec42. 00000400:00000010:1.0:1773941270.960778:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002984d6d3. 00000400:00000010:1.0:1773941270.960780:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ca631534. 00000400:00000010:1.0:1773941270.960782:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d9cc2b2e. 00000400:00000010:1.0:1773941270.960785:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000cd6beab1. 00000400:00000010:1.0:1773941270.960786:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000068abeaf. 00000400:00000010:1.0:1773941270.960787:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000f47abb19. 00000400:00000010:1.0:1773941270.960788:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009662d860. 00000400:00000010:1.0:1773941270.960789:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000730f483d. 00000400:00000010:1.0:1773941270.960791:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008e059806. 00000400:00000010:1.0:1773941270.960797:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c8f05d3e. 00000400:00000010:1.0:1773941270.960799:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000042569e01. 00000400:00000010:1.0:1773941270.960800:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000b4fdd953. 00000400:00000010:1.0:1773941270.960801:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000886d49c3. 00000400:00000010:1.0:1773941270.960802:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000030533b2. 00000400:00000010:1.0:1773941270.960804:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000074f2a88c. 00000400:00000010:1.0:1773941270.960807:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000216b3b28. 00000400:00000010:1.0:1773941270.960808:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000c265de05. 00000400:00000010:1.0:1773941270.960812:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000001bcff7f6. 00000400:00000010:1.0:1773941270.960813:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000010bc32bd. 00000400:00000010:1.0:1773941270.960814:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000eaf80efd. 00000400:00000010:1.0:1773941270.960817:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006a14c596. 00000400:00000010:1.0:1773941270.960824:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000091cb1f4a. 00000400:00000010:1.0:1773941270.960826:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000002b8157da. 00000400:00000010:1.0:1773941270.960827:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000005edbc292. 00000400:00000010:1.0:1773941270.960829:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005c657f51. 00000400:00000010:1.0:1773941270.960830:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005721d7c4. 00000400:00000010:1.0:1773941270.960835:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000048a4b5e. 00000400:00000010:1.0:1773941270.960840:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000062a765b8. 00000400:00000010:1.0:1773941270.960842:0:2335:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000ea618e0e. 00000400:00000010:1.0:1773941270.960844:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000009e43c56c. 00000400:00000010:1.0:1773941270.960845:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000cfaecb30. 00000400:00000010:1.0:1773941270.960847:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000016bfda0d. 00000400:00000010:1.0:1773941270.960852:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009405a70f. 00000400:00000010:1.0:1773941270.960862:0:2335:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000751a9607. 00000400:00000010:1.0:1773941270.960866:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000003ba5427a. 00000400:00000010:1.0:1773941270.960868:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000807c03a8. 00000400:00000010:1.0:1773941270.960870:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000deed6465. 00000400:00000010:1.0:1773941270.960874:0:2335:0:(config.c:344:lnet_net_alloc()) alloc '(net)': 176 at 0000000004484fbf. 00000400:00000010:1.0:1773941270.960877:0:2335:0:(config.c:425:lnet_ni_alloc_common()) alloc '(ni)': 448 at 000000007915ae26. 00000400:00000010:1.0:1773941270.960879:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000f59b9558. 00000400:00000010:1.0:1773941270.960880:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000184888fc. 00000400:00000010:1.0:1773941270.960881:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002b76eebf. 00000400:00000010:1.0:1773941270.960882:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000003ceae7d4. 00000400:00000010:1.0:1773941270.960883:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006ccf1144. 00000400:00000010:1.0:1773941270.960884:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000316ba7f1. 00000400:00000010:1.0:1773941270.960887:0:2335:0:(config.c:592:lnet_parse_networks()) alloc '(tokens)': 4 at 000000004a21820a. 00000400:00000010:1.0:1773941270.960891:0:2335:0:(config.c:344:lnet_net_alloc()) alloc '(net)': 176 at 0000000081faad44. 00000400:00000010:1.0:1773941270.960893:0:2335:0:(config.c:425:lnet_ni_alloc_common()) alloc '(ni)': 448 at 000000000e56f2ee. 00000400:00000010:1.0:1773941270.960895:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000890d28b9. 00000400:00000010:1.0:1773941270.960897:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001e4a5031. 00000400:00000010:1.0:1773941270.960905:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b72841b5. 00000400:00000010:1.0:1773941270.960907:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000009f3fd98c. 00000400:00000010:1.0:1773941270.960909:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e4fe5cda. 00000400:00000010:1.0:1773941270.960911:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008e8137ef. 00000400:00000010:1.0:1773941270.960913:0:2335:0:(config.c:804:lnet_parse_networks()) kfreed 'tokens': 4 at 000000004a21820a. 00000400:00000010:1.0:1773941270.960926:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000000610142d. 00000400:00000010:1.0:1773941270.960929:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 000000002eb5b41e. 00000400:00000010:1.0:1773941270.960931:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 0000000075adda96. 00000800:00000200:1.0:1773941270.961556:0:2335:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=lo ifindex 1 state 0 ns 00000000cbd44fde 00000800:00000200:1.0:1773941270.961561:0:2335:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=lo ifindex 1 state 0 ns 00000000cbd44fde 00000800:00000200:1.0:1773941270.961564:0:2335:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=ens2 ifindex 2 state 6 ns 00000000cbd44fde 00000800:00000200:1.0:1773941270.961567:0:2335:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=ens2 ifindex 2 state 6 ns 00000000cbd44fde 00000800:00000200:1.0:1773941270.961570:0:2335:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=lo ifindex 1 state 0 ns 00000000c1c06cd8 00000800:00000200:1.0:1773941270.961573:0:2335:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=lo ifindex 1 state 0 ns 00000000c1c06cd8 00000800:00000010:1.0:1773941270.961580:0:2335:0:(socklnd.c:2622:ksocknal_startup()) alloc '(net)': 208 at 000000003a53971d. 00000400:00000010:1.0:1773941270.961590:0:2335:0:(config.c:398:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 000000004a21820a. 00000800:00000200:1.0:1773941270.961595:0:2335:0:(socklnd_modparams.c:304:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1 00000800:00000010:2.0:1773941270.961778:0:2344:0:(socklnd_cb.c:1469:ksocknal_scheduler()) alloc '(rx_scratch_pgs)': 2048 at 000000003173fc79. 00000800:00000010:2.0:1773941270.961786:0:2344:0:(socklnd_cb.c:1476:ksocknal_scheduler()) alloc '(scratch_iov)': 4096 at 0000000065811064. 00000800:00000010:1.0:1773941270.961893:0:2345:0:(socklnd_cb.c:1469:ksocknal_scheduler()) alloc '(rx_scratch_pgs)': 2048 at 0000000072a30996. 00000800:00000010:1.0:1773941270.961899:0:2345:0:(socklnd_cb.c:1476:ksocknal_scheduler()) alloc '(scratch_iov)': 4096 at 0000000041aaf049. 00000800:00000010:3.0:1773941270.962001:0:2346:0:(socklnd_cb.c:1469:ksocknal_scheduler()) alloc '(rx_scratch_pgs)': 2048 at 00000000c1160c5d. 00000800:00000010:3.0:1773941270.962009:0:2346:0:(socklnd_cb.c:1476:ksocknal_scheduler()) alloc '(scratch_iov)': 4096 at 000000009819f404. 00000400:00000200:1.0:1773941270.962074:0:2335:0:(config.c:1527:lnet_get_link_status_locked()) device idx 1 get_link 2 00000800:00000010:3.0:1773941270.962082:0:2347:0:(socklnd_cb.c:1469:ksocknal_scheduler()) alloc '(rx_scratch_pgs)': 2048 at 000000004dfb10ed. 00000800:00000010:3.0:1773941270.962087:0:2347:0:(socklnd_cb.c:1476:ksocknal_scheduler()) alloc '(scratch_iov)': 4096 at 00000000a5a43bd1. 00000400:00000200:1.0:1773941270.962139:0:2335:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.204.43) 00000400:02000000:1.0:1773941270.962149:0:2335:0:(api-ni.c:2704:lnet_startup_lndni()) Added LNI 192.168.204.43@tcp [8/256/0/180] 00000400:00000010:1.0:1773941270.964396:0:2335:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at 000000008e885f99. 00000400:00000010:1.0:1773941270.964412:0:2335:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ca402c15. 00000400:00000010:1.0:1773941270.964426:0:2335:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f2de4db5. 00000400:00000200:1.0:1773941270.964437:0:2335:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 0 lazy 00000400:00000010:1.0:1773941270.964440:0:2335:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000ca5e5320. 00000400:00000010:1.0:1773941270.964443:0:2335:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009a7ffedd. 00000400:00000010:1.0:1773941270.964445:0:2335:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000fe36797e. 00000400:00000200:1.0:1773941270.964448:0:2335:0:(api-ni.c:2331:lnet_push_target_post()) posted push target 00000000ca5e5320 00000400:00000200:1.0:1773941270.964450:0:2335:0:(api-ni.c:2288:lnet_push_target_resize()) nbytes 272 success 00000400:00000010:1.0:1773941270.964457:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000095981550. 00000400:00000010:1.0:1773941270.964461:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000aef11a52. 00000400:00000010:1.0:1773941270.964463:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a345d987. 00000400:00000010:1.0:1773941270.964466:0:2335:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000818b9c89. 00000400:00000010:1.0:1773941270.964468:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000027faaccb. 00000400:00000010:1.0:1773941270.964469:0:2335:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ac29ee15. 00000400:00000200:1.0:1773941270.964757:0:2335:0:(peer.c:4215:lnet_peer_discovery_start()) discovery start: 0 00000100:00000010:1.0:1773941270.964876:0:2335:0:(ptlrpcd.c:749:ptlrpcd_init()) kmalloced '(ptlrpcds)': 16 at 0000000089ec4cc3. 00000400:00000200:0.0:1773941270.964882:0:2350:0:(peer.c:4041:lnet_peer_discovery()) started 00000100:00000010:1.0:1773941270.964998:0:2351:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000001ecd3b8. 00000020:00000010:1.0:1773941270.965005:0:2351:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000742b0f73. 00000020:00000010:1.0:1773941270.965009:0:2351:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000594c2279. 00000020:00000010:1.0:1773941270.965012:0:2351:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000446334af. 00000040:00000010:1.0:1773941270.965016:0:2351:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000b354a8e9. 00000020:00000010:1.0:1773941270.965020:0:2351:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000034758fac. 00000020:00000010:1.0:1773941270.965024:0:2351:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000bd044187. 00000100:00000010:0.0:1773941270.965041:0:2335:0:(ptlrpcd.c:846:ptlrpcd_init()) kmalloced '(pd)': 392 at 0000000035f419a6. 00000100:00000010:0.0:1773941270.965046:0:2335:0:(ptlrpcd.c:538:ptlrpcd_partners()) kmalloced '(pc->pc_partners)': 8 at 0000000097566f96. 00000100:00000010:0.0:1773941270.965049:0:2335:0:(ptlrpcd.c:538:ptlrpcd_partners()) kmalloced '(pc->pc_partners)': 8 at 0000000016f18666. 00000100:00000010:1.0:1773941270.965175:0:2352:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000007e555a3. 00000020:00000010:1.0:1773941270.965180:0:2352:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000695608ce. 00000020:00000010:1.0:1773941270.965182:0:2352:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000002741880b. 00000020:00000010:1.0:1773941270.965184:0:2352:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000cfc87e80. 00000040:00000010:1.0:1773941270.965186:0:2352:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000007087b958. 00000020:00000010:1.0:1773941270.965189:0:2352:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000fe648477. 00000020:00000010:1.0:1773941270.965192:0:2352:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000073067527. 00000100:00000010:1.0:1773941270.965323:0:2353:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000dd351f30. 00000020:00000010:1.0:1773941270.965327:0:2353:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000016d21c41. 00000020:00000010:1.0:1773941270.965330:0:2353:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000ed2443f7. 00000020:00000010:1.0:1773941270.965332:0:2353:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000e0e538f9. 00000040:00000010:1.0:1773941270.965333:0:2353:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000084d6971c. 00000020:00000010:1.0:1773941270.965336:0:2353:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000d68b6ed9. 00000020:00000010:1.0:1773941270.965339:0:2353:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000d051dd28. 00000100:00000010:0.0:1773941270.965358:0:2335:0:(ptlrpcd.c:846:ptlrpcd_init()) kmalloced '(pd)': 392 at 0000000084840526. 00000100:00000010:0.0:1773941270.965363:0:2335:0:(ptlrpcd.c:538:ptlrpcd_partners()) kmalloced '(pc->pc_partners)': 8 at 0000000012f75ad3. 00000100:00000010:0.0:1773941270.965367:0:2335:0:(ptlrpcd.c:538:ptlrpcd_partners()) kmalloced '(pc->pc_partners)': 8 at 000000007026f3be. 00000100:00000010:3.0:1773941270.965548:0:2354:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000ac78af3e. 00000020:00000010:3.0:1773941270.965553:0:2354:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000655a687e. 00000020:00000010:3.0:1773941270.965556:0:2354:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000005e6eac18. 00000020:00000010:3.0:1773941270.965559:0:2354:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 000000007e1b10b5. 00000040:00000010:3.0:1773941270.965562:0:2354:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000e174c96d. 00000020:00000010:3.0:1773941270.965565:0:2354:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000092de8d7. 00000020:00000010:3.0:1773941270.965567:0:2354:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005a3b7c3a. 00000100:00000010:3.0:1773941270.965721:0:2355:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000ef2e4021. 00000020:00000010:3.0:1773941270.965726:0:2355:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000f03c48ff. 00000020:00000010:3.0:1773941270.965729:0:2355:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000f3b7306a. 00000020:00000010:3.0:1773941270.965731:0:2355:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000d241189b. 00000040:00000010:3.0:1773941270.965734:0:2355:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000574ff331. 00000020:00000010:3.0:1773941270.965737:0:2355:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000002633a601. 00000020:00000010:3.0:1773941270.965739:0:2355:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000046cfea82. 00000100:00000010:0.0:1773941270.965788:0:2335:0:(connection.c:186:ptlrpc_connection_init()) kmalloced '(cpus_latency_qos)': 1088 at 000000009003c438. 00000100:00000010:0.0:1773941270.965911:0:2335:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 00000000de1af156. 00000100:00000010:0.0:1773941270.965915:0:2335:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 00000000325f3f0b. 00000100:00000010:0.0:1773941270.965917:0:2335:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 000000005a6a2f99. 00000100:00000010:0.0:1773941270.965918:0:2335:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 0000000068843031. 00000100:00000010:0.0:1773941270.965920:0:2335:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 000000006dbd9efd. 00000100:00000010:0.0:1773941270.965921:0:2335:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 00000000a91fe00c. 00000001:00000010:0.0:1773941270.966040:0:2335:0:(nodemap_handler.c:4695:nodemap_config_alloc()) kmalloced '(config)': 480 at 000000000ff56e80. 00000001:00000010:0.0:1773941270.966044:0:2335:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 000000006b0c0c2f. 00000001:00000010:0.0:1773941270.966047:0:2335:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 8 at 00000000841b23b3. 00000001:00000010:0.0:1773941270.966049:0:2335:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 152 at 00000000b47a2c50. 00000001:00000010:0.0:1773941270.966056:0:2335:0:(nodemap_handler.c:3541:nodemap_create()) kmalloced '(nodemap)': 968 at 000000008cc12191. 00000001:00000010:0.0:1773941270.966088:0:2335:0:(nodemap_lproc.c:1443:lprocfs_nodemap_register()) kmalloced '(nm_entry)': 48 at 000000003ef3d80a. 02000000:00000010:3.0:1773941271.037142:0:2367:0:(upcall_cache.c:573:upcall_cache_init()) kmalloced '(cache)': 1208 at 000000007161e94e. 02000000:00000010:3.0:1773941271.037150:0:2367:0:(upcall_cache.c:581:upcall_cache_init()) kmalloced '(cache->uc_hashtable)': 1024 at 00000000203c3b12. 02000000:00000010:3.0:1773941271.037154:0:2367:0:(upcall_cache.c:573:upcall_cache_init()) kmalloced '(cache)': 1208 at 000000002eaedb29. 02000000:00000010:3.0:1773941271.037164:0:2367:0:(upcall_cache.c:581:upcall_cache_init()) kmalloced '(cache->uc_hashtable)': 16384 at 00000000042f92eb. 00000020:00000010:2.0:1773941272.759908:0:2419:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000dc78ea54. 00000001:00000010:2.0:1773941272.840955:0:2432:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 0000000057de1b80. 00000100:00000010:2.0:1773941272.840987:0:2432:0:(client.c:689:ptlrpc_init_rq_pool()) kmalloced '(pool)': 56 at 00000000bacdc999. 00000020:00000010:2.0:1773941272.841015:0:2432:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000f19637dc. 00000020:00000010:2.0:1773941272.841034:0:2432:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000bb7ce105. 00000020:00000010:2.0:1773941272.841042:0:2432:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000bb7ce105. 00000020:00000010:3.0:1773941272.905988:0:2445:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000dcf6ce6c. 00000020:00000010:3.0:1773941272.960562:0:2458:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000233ec6fc. 00000020:00000010:3.0:1773941273.001678:0:2472:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000fb80e543. 00008000:02000000:3.0:1773941273.043905:0:2485:0:(echo_client.c:2570:obdecho_init()) Echo OBD driver; http://www.lustre.org/ 00000020:00000010:3.0:1773941273.045859:0:2485:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000e9a2bb9f. 00000020:00000010:3.0:1773941273.046092:0:2485:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000951a2c75. 00000020:00000010:3.0:1773941273.197477:0:2508:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000009c4fd015. 00000020:00000010:3.0:1773941273.790758:0:2803:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000006728490a. 00000020:00000010:3.0:1773941273.864310:0:2816:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000003d059624. 00000020:00000010:3.0:1773941273.986890:0:2829:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000493d11e6. 00000020:00000010:3.0:1773941273.986916:0:2829:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000003298532c. 00000020:00000010:2.0:1773941274.094829:0:2850:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000004188c8d7. 00000020:00000010:3.0:1773941274.164122:0:2863:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000bfef372a. 00000020:00000010:3.0:1773941274.164364:0:2863:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000c51be981. 00000020:00000010:0.0:1773941274.237725:0:2878:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000004f17eb7f. 00000020:00000010:0.0:1773941274.237742:0:2878:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000f8ab675b. 00000020:00000010:0.0:1773941274.372450:0:2891:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at 000000004a8f6776. 00000020:00000010:0.0:1773941274.372460:0:2891:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000cb6111b6. 00000020:00000010:0.0:1773941274.372463:0:2891:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000052fe692d. 00000020:00000010:0.0:1773941274.372466:0:2891:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 0000000082e038c7. 00000008:00000010:0.0:1773941274.372479:0:2891:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000c73e4720. 00020000:00000010:0.0:1773941274.372483:0:2891:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000007943175d. 00008000:00000010:0.0:1773941274.372493:0:2891:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000001689293a. 00000080:00000010:0.0:1773941274.372504:0:2891:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000f4ac18bf. 00000080:00000010:0.0:1773941274.372515:0:2891:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 0000000008c0f2b8. 00000020:00000010:0.0:1773941274.372519:0:2891:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005c0d92cf. 00000008:00000010:0.0:1773941274.372528:0:2891:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000c6dab39b. 00020000:00000010:0.0:1773941274.372537:0:2891:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 0000000020d2751f. 00008000:00000010:0.0:1773941274.372553:0:2891:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 0000000054acf110. 00000080:00000010:0.0:1773941274.372562:0:2891:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 00000000ecac39f1. 00000800:00000200:2.0:1773941276.857085:0:37:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_GOING_DOWN, iface=lo ifindex 1 state 0 ns 00000000c1c06cd8 00000800:00000200:2.0:1773941276.857464:0:37:0:(socklnd.c:2209:ksocknal_inet6addr_event()) addr6event: status NETDEV_DOWN, device lo, ip addr (einval), ns 00000000c1c06cd8 00000800:00000200:2.0:1773941276.857471:0:37:0:(socklnd.c:2112:ksocknal_handle_inetaddr_change()) Skipping device lo from namespace 00000000c1c06cd8 (expected 00000000cbd44fde) 00000800:00000200:2.0:1773941276.857483:0:37:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_DOWN, iface=lo ifindex 1 state 0 ns 00000000c1c06cd8 00000800:00000200:2.0:1773941276.857487:0:37:0:(socklnd.c:1972:ksocknal_handle_link_state_change()) Skipping device lo from namespace 00000000c1c06cd8 (expected 00000000cbd44fde) 00000800:00000200:2.0:1773941276.857674:0:37:0:(socklnd.c:2179:ksocknal_inetaddr_event()) addrevent: status NETDEV_DOWN device lo, ip addr 127.0.0.1, netmask 255.0.0.0 ns 00000000c1c06cd8. 00000800:00000200:2.0:1773941276.857690:0:37:0:(socklnd.c:2112:ksocknal_handle_inetaddr_change()) Skipping device lo from namespace 00000000c1c06cd8 (expected 00000000cbd44fde) 00000800:00000200:2.0:1773941276.857935:0:37:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UNREGISTER, iface=lo ifindex 1 state 0 ns 00000000c1c06cd8 00000800:00000200:2.0:1773941276.857939:0:37:0:(socklnd.c:1972:ksocknal_handle_link_state_change()) Skipping device lo from namespace 00000000c1c06cd8 (expected 00000000cbd44fde) 00000080:00000010:0.0:1773941319.281153:0:5414:0:(super25.c:253:lustre_init_fs_context()) kmalloced '(lmd)': 128 at 0000000059e29040. 00000020:00000010:0.0:1773941319.281164:0:5414:0:(obd_mount.c:1388:lustre_parse_monolithic()) kmalloced '(lmd->lmd_params)': 4096 at 00000000bed9502b. 00000020:00000010:0.0:1773941319.281173:0:5414:0:(obd_mount.c:1590:lustre_parse_monolithic()) kmalloced '(lmd->lmd_dev)': 28 at 00000000553e16d3. 00000020:00000010:0.0:1773941319.281176:0:5414:0:(obd_mount.c:1646:lustre_parse_monolithic()) kmalloced '(lmd->lmd_profile)': 14 at 0000000049b9e0f5. 00000020:00000010:0.0:1773941319.281178:0:5414:0:(obd_mount.c:1674:lustre_parse_monolithic()) kmalloced '(lmd->lmd_opts)': 17 at 00000000ed2aaa6a. 00000020:00000010:0.0:1773941319.281329:0:5414:0:(obd_mount.c:656:lustre_init_lsi()) kmalloced '(lsi)': 1880 at 000000004259c4fc. 00000020:00000010:0.0:1773941319.281346:0:5414:0:(obd_mount.c:345:lustre_start_mgc()) kmalloced '(mgcname)': 23 at 00000000b34fa120. 00000020:00000010:0.0:1773941319.281349:0:5414:0:(obd_mount.c:353:lustre_start_mgc()) kmalloced '(data)': 192 at 00000000bd9a295e. 00000020:00000010:0.0:1773941319.281356:0:5414:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 88 at 00000000a50b164a. 00000100:00000010:0.0:1773941319.281361:0:5414:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at 0000000073ff3401. 00000400:00000200:0.0:1773941319.281365:0:5414:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:0.0:1773941319.281368:0:5414:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.143@tcp 00000400:00000200:0.0:1773941319.281373:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941319.281379:0:5414:0:(peer.c:229:lnet_peer_alloc()) alloc '(lp)': 376 at 00000000cbaeb348. 00000400:00000200:0.0:1773941319.281382:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.281384:0:5414:0:(peer.c:262:lnet_peer_alloc()) 00000000cbaeb348 nid 192.168.204.143@tcp 00000400:00000010:0.0:1773941319.281388:0:5414:0:(peer.c:193:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 000000004f772651. 00000400:00000200:0.0:1773941319.281395:0:5414:0:(peer.c:202:lnet_peer_net_alloc()) 000000004f772651 net tcp 00000400:00000200:0.0:1773941319.281398:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941319.281401:0:5414:0:(peer.c:137:lnet_peer_ni_alloc()) alloc '(lpni)': 384 at 000000006ecb0717. 00000400:00000200:0.0:1773941319.281404:0:5414:0:(peer.c:183:lnet_peer_ni_alloc()) 000000006ecb0717 nid 192.168.204.143@tcp 00000400:00000200:0.0:1773941319.281410:0:5414:0:(peer.c:1091:lnet_peer_ni_clr_non_mr_pref_nid()) peer 192.168.204.143@tcp: -2 00000400:00000200:0.0:1773941319.281415:0:5414:0:(peer.c:1639:lnet_peer_attach_peer_ni()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp flags 0x100001 00000400:00000200:0.0:1773941319.281420:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.281440:0:5414:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.143@tcp: 0 00000400:00000200:0.0:1773941319.281442:0:5414:0:(peer.c:2727:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:0.0:1773941319.281444:0:5414:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941319.281446:0:5414:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp: 0. pending discovery 00000020:00000010:0.0:1773941319.281450:0:5414:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 88 at 00000000a50b164a. 00000020:00000010:0.0:1773941319.281455:0:5414:0:(obd_mount.c:487:lustre_start_mgc()) kmalloced '(uuid)': 40 at 0000000032618b06. 00000020:00000010:0.0:1773941319.281460:0:5414:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 120 at 000000005ab7168d. 00000020:00000010:0.0:1773941319.281480:0:5414:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at 000000000c65c0ec. 00000400:00000200:1.0:1773941319.281483:0:2350:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000020:00000010:0.0:1773941319.281487:0:5414:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 00000000cc1dd02f. 00000400:00000200:1.0:1773941319.281492:0:2350:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.143@tcp(00000000cbaeb348) state 0x100041 00000400:00000010:1.0:1773941319.281499:0:2350:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 00000000a18edf7e. 00000400:00000010:1.0:1773941319.281504:0:2350:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000056039027. 00000020:00000010:0.0:1773941319.281517:0:5414:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 120 at 000000005ab7168d. 00000020:00000010:0.0:1773941319.281523:0:5414:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 104 at 000000005ab7168d. 00000400:00000010:1.0:1773941319.281529:0:2350:0:(lib-lnet.h:478:lnet_rspt_alloc()) alloc 'rspt': 56 at 000000005f68a9d2. 00000400:00000200:1.0:1773941319.281535:0:2350:0:(lib-move.c:5547:LNetGet()) LNetGet -> 12345-192.168.204.143@tcp 00000001:00000010:0.0:1773941319.281545:0:5414:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 000000006a705fa0. 00000400:00000200:1.0:1773941319.281549:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000001:00000010:0.0:1773941319.281551:0:5414:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 00000000f00de5a4. 00000400:00000200:1.0:1773941319.281557:0:2350:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000001:00000010:0.0:1773941319.281557:0:5414:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000006b89150b. 00000001:00000010:0.0:1773941319.281562:0:5414:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000005e6afdab. 00000400:00000200:1.0:1773941319.281565:0:2350:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000001:00000010:0.0:1773941319.281568:0:5414:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000007e3720e8. 00000400:00000200:1.0:1773941319.281572:0:2350:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000001:00000010:0.0:1773941319.281574:0:5414:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000c8a45c84. 00000400:00000200:1.0:1773941319.281577:0:2350:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941319.281581:0:2350:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000020:00000010:0.0:1773941319.281583:0:5414:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000001df3009d. 00000400:00000200:1.0:1773941319.281587:0:2350:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [1:1:256:4294967295] 192.168.204.143@tcp LPNI seq info [1:1:8:4294967295] 00000020:00000010:0.0:1773941319.281590:0:5414:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000000a4a0442. 00000400:00000200:1.0:1773941319.281594:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941319.281600:0:2350:0:(lib-move.c:2009:lnet_handle_send()) rspt_next_hop_nid = 192.168.204.143@tcp 00000400:00000200:1.0:1773941319.281606:0:2350:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : GET try# 0 00000020:00000010:0.0:1773941319.281606:0:5414:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000097b0e1b6. 00000800:00000200:1.0:1773941319.281615:0:2350:0:(socklnd_cb.c:1001:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.204.143@tcp 00000008:00000010:0.0:1773941319.281616:0:5414:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 000000000ecb35c1. 00000800:00000010:1.0:1773941319.281622:0:2350:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 000000000f848b34. 00020000:00000010:0.0:1773941319.281623:0:5414:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 000000008b33d792. 00008000:00000010:0.0:1773941319.281628:0:5414:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 00000000a3927956. 00000080:00000010:0.0:1773941319.281633:0:5414:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 0000000015ae5784. 00000020:00000010:0.0:1773941319.281639:0:5414:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000011b24032. 00000400:00000200:1.0:1773941319.281642:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000010:0.0:1773941319.281643:0:5414:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000e20d0e0f. 00000800:00000010:1.0:1773941319.281648:0:2350:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 000000007a1c9c6c. 00000800:00000010:1.0:1773941319.281655:0:2350:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 00000000a4c2fa4c. 00000040:00000010:0.0:1773941319.281656:0:5414:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000002c125ee6. 00000800:00000200:1.0:1773941319.281661:0:2350:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (2) 00000020:00000010:0.0:1773941319.281663:0:5414:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000009257afff. 00000800:00000200:1.1F:1773941319.281668:0:2350:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (2) 00000020:00000010:0.0:1773941319.281669:0:5414:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 0000000020f5eef1. 00040000:00000010:0.0:1773941319.281675:0:5414:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000003d805dca. 00040000:00000010:0.0:1773941319.281681:0:5414:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000555bde7e. 10000000:00000010:0.0:1773941319.281685:0:5414:0:(mgc_request.c:852:mgc_device_alloc()) kmalloced '(lu)': 64 at 000000004cf83b02. 00000400:00000200:1.0:1773941319.281692:0:2350:0:(peer.c:3788:lnet_peer_send_ping()) peer 192.168.204.143@tcp 00000400:00000200:1.0:1773941319.281695:0:2350:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.143@tcp(00000000cbaeb348) state 0x100241 rc 0 00010000:00000010:0.0:1773941319.281695:0:5414:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at 00000000c43ad1da. 00010000:00000010:0.0:1773941319.281753:0:5414:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at 000000008a132a27. 00010000:00000010:0.0:1773941319.281765:0:5414:0:(ldlm_lockd.c:3356:ldlm_setup()) kmalloced '(ldlm_state)': 32 at 00000000f8f380c8. 00000100:00000010:0.0:1773941319.281839:0:5414:0:(service.c:745:ptlrpc_register_service()) kmalloced '(service)': 360 at 0000000068c96e1f. 00000100:00000010:0.0:1773941319.281844:0:5414:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at 0000000032bf57f5. 00000100:00000010:0.0:1773941319.281857:0:5414:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at 0000000043931b91. 00000100:00000010:0.0:1773941319.281864:0:5414:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at 00000000058e8b95. 00000100:00000010:0.0:1773941319.281868:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c5ac7a6d. 00000100:00000010:0.0:1773941319.281871:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000082b9c9b. 00000100:00000010:0.0:1773941319.281873:0:5414:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at 000000006f83ca11. 00000100:00000010:0.0:1773941319.281881:0:5414:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at 0000000052526a8f. 00000100:00000010:0.0:1773941319.281885:0:5414:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at 000000001721a033. 00000100:00000010:0.0:1773941319.281888:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000000fc42274. 00000100:00000010:0.0:1773941319.281891:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000073533e67. 00000400:00000200:0.0:1773941319.281894:0:5414:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 15 lazy 00000020:00000010:0.0:1773941319.281918:0:5414:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 000000005b062384. 00000020:00000010:0.0:1773941319.281921:0:5414:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 000000005fec10ce. 00000100:00000010:0.0:1773941319.281949:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000010a2d066. 00000100:00000010:0.0:1773941319.281953:0:5414:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 00000000bb23e866. 00000100:00000010:0.0:1773941319.281955:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000007453cee0. 00000100:00000010:0.0:1773941319.281958:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000021800680. 00000100:00000010:0.0:1773941319.281959:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000107433d7. 00000100:00000010:0.0:1773941319.281961:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000c767fcdd. 00000100:00000010:0.0:1773941319.281962:0:5414:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 00000000f6ebac3d. 00000100:00000010:0.0:1773941319.281964:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000e80ff7a6. 00000100:00000010:0.0:1773941319.281969:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000047acdf41. 00000100:00000010:0.0:1773941319.281970:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000b7d06752. 00000100:00000200:0.0:1773941319.281990:0:5414:0:(service.c:848:ptlrpc_register_service()) ldlm_cbd: Started, listening on portal 15 00000100:00000010:0.0:1773941319.281995:0:5414:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 00000000a6dfed3c. 00000100:00000010:0.0:1773941319.282269:0:5415:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 000000007c67653b. 00000020:00000010:0.0:1773941319.282275:0:5415:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000e24a14b4. 00000020:00000010:0.0:1773941319.282280:0:5415:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000000472bc3. 00000020:00000010:0.0:1773941319.282283:0:5415:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000003a1334a7. 00000040:00000010:0.0:1773941319.282286:0:5415:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000004aa9e635. 00000020:00000010:0.0:1773941319.282289:0:5415:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000006c627962. 00000020:00000010:0.0:1773941319.282304:0:5415:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000f236dd9c. 00000020:00000010:0.0:1773941319.282309:0:5415:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000e9e15ebf. 80000000:00000010:0.0:1773941319.282313:0:5415:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 000000005c12d272. 40000000:00000010:0.0:1773941319.282316:0:5415:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000c07c5161. 00100000:00000010:0.0:1773941319.282321:0:5415:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 0000000053c1c6b7. 00040000:00000010:0.0:1773941319.282324:0:5415:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000435f5212. 00040000:00000010:0.0:1773941319.282327:0:5415:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000c51fc78c. 00040000:00000010:0.0:1773941319.282330:0:5415:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000003d74306a. 00000004:00000010:0.0:1773941319.282333:0:5415:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000d07b624b. 00000004:00000010:0.0:1773941319.282339:0:5415:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 00000000d2510c2e. 00000004:00000010:0.0:1773941319.282343:0:5415:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 0000000073ef5b6c. 00002000:00000010:0.0:1773941319.282347:0:5415:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000fbc9f552. 00000004:00000010:0.0:1773941319.282351:0:5415:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 0000000058b9d29a. 00000100:00000200:0.0:1773941319.282355:0:5415:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_cbd: registering portal 15 00000400:00000010:0.0:1773941319.282361:0:5415:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000862132d3. 00000400:00000010:0.0:1773941319.282366:0:5415:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000db9ff3ca. 00000100:00000010:0.0:1773941319.282375:0:5415:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 00000000372726fb. 00000100:00000200:0.0:1773941319.282392:0:5415:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:2.0:1773941319.282413:0:5414:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 00000000e7ca7da3. 00000800:00000010:3.0:1773941319.282585:0:2339:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4712 at 000000007b1b6dc0. 00000800:00000010:3.0:1773941319.282596:0:2339:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 000000009a23504b. 00000800:00000010:3.0:1773941319.282604:0:2339:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000d38a5e31. 00000100:00000010:1.0:1773941319.282659:0:5416:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 000000009914b446. 00000020:00000010:1.0:1773941319.282663:0:5416:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000d5acd892. 00000020:00000010:1.0:1773941319.282666:0:5416:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006e548109. 00000800:00000010:3.0:1773941319.282669:0:2339:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000d38a5e31. 00000020:00000010:1.0:1773941319.282670:0:5416:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000cd235794. 00000040:00000010:1.0:1773941319.282680:0:5416:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000dae0d3ec. 00000020:00000010:1.0:1773941319.282684:0:5416:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000cc1cdaba. 00000020:00000010:1.0:1773941319.282696:0:5416:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000f9d53695. 00000020:00000010:1.0:1773941319.282698:0:5416:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000008865145c. 80000000:00000010:1.0:1773941319.282700:0:5416:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000d16084d0. 40000000:00000010:1.0:1773941319.282721:0:5416:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 000000002d235d27. 00100000:00000010:1.0:1773941319.282726:0:5416:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000001fc02f83. 00040000:00000010:1.0:1773941319.282730:0:5416:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000d92064aa. 00040000:00000010:1.0:1773941319.282737:0:5416:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000f8960bbf. 00040000:00000010:1.0:1773941319.282740:0:5416:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000005979ab63. 00000004:00000010:1.0:1773941319.282744:0:5416:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 0000000063410063. 00000004:00000010:1.0:1773941319.282747:0:5416:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 000000004d419d88. 00000004:00000010:1.0:1773941319.282750:0:5416:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000afc0c383. 00002000:00000010:1.0:1773941319.282753:0:5416:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 0000000014d84a6b. 00000004:00000010:1.0:1773941319.282755:0:5416:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 0000000097ad28af. 00000100:00000010:1.0:1773941319.282758:0:5416:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 000000003836d9d7. 00000100:00000200:1.0:1773941319.282772:0:5416:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:2.0:1773941319.282791:0:5414:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 0000000042c65a7f. 00000100:00000010:2.0:1773941319.282987:0:5417:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 0000000080ed0e1b. 00000020:00000010:2.0:1773941319.282990:0:5417:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 0000000003feb69a. 00000020:00000010:2.0:1773941319.282993:0:5417:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000b50954d. 00000020:00000010:2.0:1773941319.282994:0:5417:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000c949ef49. 00000040:00000010:2.0:1773941319.282996:0:5417:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000f9f10926. 00000020:00000010:2.0:1773941319.282998:0:5417:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000829d5fd7. 00000020:00000010:2.0:1773941319.283008:0:5417:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 000000001023ccfc. 00000020:00000010:2.0:1773941319.283011:0:5417:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000006535d8e6. 80000000:00000010:2.0:1773941319.283012:0:5417:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 0000000086d67a08. 40000000:00000010:2.0:1773941319.283014:0:5417:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 000000009a120489. 00100000:00000010:2.0:1773941319.283016:0:5417:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000001b26082c. 00040000:00000010:2.0:1773941319.283017:0:5417:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000872da2a5. 00040000:00000010:2.0:1773941319.283019:0:5417:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000e8db11b7. 00040000:00000010:2.0:1773941319.283020:0:5417:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000001428ed23. 00000004:00000010:2.0:1773941319.283022:0:5417:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000afe94590. 00000004:00000010:2.0:1773941319.283025:0:5417:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 000000004f442947. 00000004:00000010:2.0:1773941319.283026:0:5417:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000294f219a. 00002000:00000010:2.0:1773941319.283028:0:5417:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000f0c1fed5. 00000004:00000010:2.0:1773941319.283029:0:5417:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 0000000074fe652a. 00000100:00000200:2.0:1773941319.283031:0:5417:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_cbd: registering portal 15 00000400:00000010:2.0:1773941319.283034:0:5417:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004b36a90b. 00000400:00000010:2.0:1773941319.283037:0:5417:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000048461262. 00000100:00000010:2.0:1773941319.283040:0:5417:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 00000000731fd05f. 00000100:00000200:2.0:1773941319.283049:0:5417:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:0.0:1773941319.283059:0:5414:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 0000000063f28ad3. 00000400:00000200:0.0:1773941319.283286:0:2339:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:0.1F:1773941319.283291:0:2339:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:0.1:1773941319.283304:0:2339:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.1:1773941319.283309:0:2339:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000000f848b34 type 2, nob 96 niov 1 nkiov 0 00000800:00000200:0.0:1773941319.283315:0:2339:0:(socklnd.c:1183:ksocknal_create_conn()) New conn 12345-192.168.204.143@tcp p 3.x 192.168.204.43 -> 192.168.204.143:988 incarnation:1773941276392337894 sched[1] 00000800:00000010:0.0:1773941319.283318:0:2339:0:(socklnd.c:1191:ksocknal_create_conn()) kfreed 'hello': 144 at 000000009a23504b. 00000400:00000200:0.0:1773941319.283369:0:2339:0:(lib-socket.c:453:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.204.143:988 00000100:00000010:3.0:1773941319.283394:0:5418:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 00000000d38a5e31. 00000800:00000200:2.0:1773941319.283397:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000020:00000010:3.0:1773941319.283398:0:5418:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000004bdc2f10. 00000800:00000010:2.0:1773941319.283402:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 000000000f848b34. 00000020:00000010:3.0:1773941319.283403:0:5418:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e76881e4. 00000400:00000200:2.0:1773941319.283408:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:3.0:1773941319.283409:0:5418:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000008cbb410b. 00000040:00000010:3.0:1773941319.283412:0:5418:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000de8b7b5f. 00000020:00000010:3.0:1773941319.283415:0:5418:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000ad0b0a61. 00000400:00000200:2.0:1773941319.283419:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: GET: OK 00000400:00000200:2.0:1773941319.283424:0:2347:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1773941319.283426:0:2347:0:(peer.c:3033:lnet_discovery_event_send()) Ping Send to 192.168.204.143@tcp: 0 00000020:00000010:3.0:1773941319.283428:0:5418:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000c8973bf1. 00000020:00000010:3.0:1773941319.283431:0:5418:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000119244bb. 80000000:00000010:3.0:1773941319.283434:0:5418:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000c89acdc5. 40000000:00000010:3.0:1773941319.283436:0:5418:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000515830c1. 00100000:00000010:3.0:1773941319.283439:0:5418:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000eeebf6e0. 00040000:00000010:3.0:1773941319.283442:0:5418:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000000309d1bc. 00040000:00000010:3.0:1773941319.283503:0:5418:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 0000000073af0861. 00040000:00000010:3.0:1773941319.283506:0:5418:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000003fad611a. 00000004:00000010:3.0:1773941319.283509:0:5418:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000006072bdbe. 00000004:00000010:3.0:1773941319.283512:0:5418:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 00000000e1299d4f. 00000004:00000010:3.0:1773941319.283514:0:5418:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000aa28ac78. 00002000:00000010:3.0:1773941319.283516:0:5418:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000d18e0083. 00000004:00000010:3.0:1773941319.283519:0:5418:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 0000000085e6b41b. 00000100:00000010:3.0:1773941319.283527:0:5418:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 00000000f85ea785. 00000100:00000200:3.0:1773941319.283540:0:5418:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:1.0:1773941319.283569:0:5414:0:(service.c:745:ptlrpc_register_service()) kmalloced '(service)': 360 at 00000000d8173c03. 00000100:00000010:1.0:1773941319.283575:0:5414:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at 000000009c10c198. 00000100:00000010:1.0:1773941319.283587:0:5414:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at 000000000f53b343. 00000100:00000010:1.0:1773941319.283594:0:5414:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at 00000000bb935e25. 00000100:00000010:1.0:1773941319.283599:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000044aaabc8. 00000100:00000010:1.0:1773941319.283603:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b2fbcadf. 00000100:00000010:1.0:1773941319.283605:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e7ffb5dc. 00000100:00000010:1.0:1773941319.283609:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a4667d39. 00000100:00000010:1.0:1773941319.283611:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000000467f97d. 00000100:00000010:1.0:1773941319.283614:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a9d119c7. 00000100:00000010:1.0:1773941319.283617:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000008cd01eec. 00000100:00000010:1.0:1773941319.283626:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b5905032. 00000100:00000010:1.0:1773941319.283630:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003c6f0ef9. 00000100:00000010:1.0:1773941319.283633:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ad2ea2de. 00000100:00000010:1.0:1773941319.283636:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000000b9ff9d4. 00000100:00000010:1.0:1773941319.283639:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000535d6b83. 00000100:00000010:1.0:1773941319.283641:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000007fa67b70. 00000100:00000010:1.0:1773941319.283645:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000058a4ad7b. 00000100:00000010:1.0:1773941319.283647:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000001e8b5308. 00000100:00000010:1.0:1773941319.283656:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000034952109. 00000100:00000010:1.0:1773941319.283659:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000df012b12. 00000100:00000010:1.0:1773941319.283662:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000959c7846. 00000100:00000010:1.0:1773941319.283665:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000a5eb1724. 00000100:00000010:1.0:1773941319.283673:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000030f6846d. 00000100:00000010:1.0:1773941319.283676:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f706b043. 00000100:00000010:1.0:1773941319.283680:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000000062c94b. 00000100:00000010:1.0:1773941319.283683:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000033487154. 00000100:00000010:1.0:1773941319.283692:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b000c0ba. 00000100:00000010:1.0:1773941319.283695:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006e7b4033. 00000100:00000010:1.0:1773941319.283698:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000efd3d30a. 00000100:00000010:1.0:1773941319.283701:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000062682d4a. 00000100:00000010:1.0:1773941319.283732:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000006cac5b42. 00000100:00000010:1.0:1773941319.283736:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000073081947. 00000100:00000010:1.0:1773941319.283740:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000006e6d7d06. 00000100:00000010:1.0:1773941319.283743:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b444ee5d. 00000100:00000010:1.0:1773941319.283754:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000005c6444ad. 00000100:00000010:1.0:1773941319.283758:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006bd557ee. 00000100:00000010:1.0:1773941319.283763:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000008ae5f06c. 00000100:00000010:1.0:1773941319.283765:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000023bb1084. 00000100:00000010:1.0:1773941319.283769:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000006c0da624. 00000100:00000010:1.0:1773941319.283773:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000033e39e86. 00000100:00000010:1.0:1773941319.283778:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000040598617. 00000100:00000010:1.0:1773941319.283782:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ecc2bdff. 00000100:00000010:1.0:1773941319.283791:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a16d7c56. 00000100:00000010:1.0:1773941319.283795:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000403454a0. 00000800:00000010:0.0:1773941319.283799:0:2339:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4712 at 00000000fd716ede. 00000100:00000010:1.0:1773941319.283800:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000003ba5b756. 00000800:00000010:0.0:1773941319.283806:0:2339:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 000000009faddc4e. 00000100:00000010:1.0:1773941319.283807:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000057e397e5. 00000100:00000010:1.0:1773941319.283814:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000005b007f21. 00000800:00000010:0.0:1773941319.283814:0:2339:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 000000007b0f354c. 00000100:00000010:1.0:1773941319.283818:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c40c6077. 00000100:00000010:1.0:1773941319.283824:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000007daa1b70. 00000800:00000010:0.0:1773941319.283827:0:2339:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 000000007b0f354c. 00000100:00000010:1.0:1773941319.283828:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000eb9aba11. 00000100:00000010:1.0:1773941319.283840:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d76e1bcd. 00000100:00000010:1.0:1773941319.283844:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006fbe8d5f. 00000100:00000010:1.0:1773941319.283847:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000dfa44481. 00000100:00000010:1.0:1773941319.283854:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b66f61fb. 00000100:00000010:1.0:1773941319.283857:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000128fb08a. 00000100:00000010:1.0:1773941319.283859:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000002762dcd7. 00000100:00000010:1.0:1773941319.283863:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000020ee99d9. 00000100:00000010:1.0:1773941319.283865:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000007e91bcae. 00000100:00000010:1.0:1773941319.283873:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000008bd6a410. 00000100:00000010:1.0:1773941319.283876:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e40be7bb. 00000100:00000010:1.0:1773941319.283879:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ef08d301. 00000100:00000010:1.0:1773941319.283882:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f0ef27da. 00000100:00000010:1.0:1773941319.283885:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000f8e4a6b0. 00000100:00000010:1.0:1773941319.283888:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000cbd1e8b9. 00000100:00000010:1.0:1773941319.283892:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000bd8c7e06. 00000100:00000010:1.0:1773941319.283895:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000002825ed46. 00000100:00000010:1.0:1773941319.283912:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000002c54f7f7. 00000100:00000010:1.0:1773941319.283917:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000bdabd918. 00000100:00000010:1.0:1773941319.283921:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000323a4cae. 00000100:00000010:1.0:1773941319.283924:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000000e43c325. 00000100:00000010:1.0:1773941319.283928:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000004c40626c. 00000100:00000010:1.0:1773941319.283931:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000034c4a57b. 00000100:00000010:1.0:1773941319.283935:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000071c5056b. 00000100:00000010:1.0:1773941319.283939:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003969231f. 00000100:00000010:1.0:1773941319.283949:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000002c62ecbb. 00000100:00000010:1.0:1773941319.283953:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000077cce7c8. 00000400:00000200:3.0:1773941319.283955:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.283961:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : REPLY - for me 00000100:00000010:1.0:1773941319.283962:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000442d1bf5. 00000100:00000010:1.0:1773941319.283968:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c91b0c92. 00000100:00000010:1.0:1773941319.283972:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000052234e28. 00000100:00000010:1.0:1773941319.283974:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000030eb35b8. 00000100:00000010:1.0:1773941319.283978:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a947e880. 00000400:00000200:3.0:1773941319.283980:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.283985:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941319.283985:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000373dd5ea. 00000400:00000200:3.0:1773941319.283992:0:2346:0:(lib-move.c:4647:lnet_parse_reply()) 192.168.204.43@tcp: Reply from 12345-192.168.204.143@tcp of length 48/48 into md 0x11 00000100:00000010:1.0:1773941319.283998:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d7fd147b. 00000100:00000010:1.0:1773941319.284002:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000080b4d3e8. 00000400:00000200:3.0:1773941319.284003:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.284009:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: REPLY: OK 00000100:00000010:1.0:1773941319.284012:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000002b405371. 00000400:00000200:3.0:1773941319.284014:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000056039027 00000100:00000010:1.0:1773941319.284016:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003d41785d. 00000400:00000010:3.0:1773941319.284017:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 0000000056039027. 00000400:00000200:3.0:1773941319.284022:0:2346:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 3 00000400:00000200:3.0:1773941319.284025:0:2346:0:(peer.c:2911:lnet_discovery_event_reply()) Peer 192.168.204.143@tcp has discovery enabled 00000400:00000200:3.0:1773941319.284027:0:2346:0:(peer.c:2923:lnet_discovery_event_reply()) peer 192.168.204.143@tcp(00000000cbaeb348) is MR 00000400:00000200:3.0:1773941319.284030:0:2346:0:(peer.c:2992:lnet_discovery_event_reply()) peer 192.168.204.143@tcp data present 363512030. state = 0x100241 00000100:00000010:1.0:1773941319.284030:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000008e802639. 00000100:00000010:1.0:1773941319.284035:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c566e5fd. 00000400:00000200:3.0:1773941319.284037:0:2346:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.143@tcp: -114 00000100:00000010:1.0:1773941319.284042:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d40b50ba. 00000100:00000010:1.0:1773941319.284048:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000007d5b4ab8. 00000100:00000010:1.0:1773941319.284057:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000043115b4. 00000100:00000010:1.0:1773941319.284061:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000018f13252. 00000100:00000010:1.0:1773941319.284065:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000048bfb3a3. 00000100:00000010:1.0:1773941319.284069:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e59e1418. 00000100:00000010:1.0:1773941319.284073:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000cc279f36. 00000100:00000010:1.0:1773941319.284077:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000fc031346. 00000100:00000010:1.0:1773941319.284081:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000aab95ad6. 00000100:00000010:1.0:1773941319.284084:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ee3425e7. 00000100:00000010:1.0:1773941319.284093:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d5693594. 00000100:00000010:1.0:1773941319.284097:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000001452d744. 00000100:00000010:1.0:1773941319.284101:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d777c31b. 00000100:00000010:1.0:1773941319.284111:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000d9b58167. 00000100:00000010:1.0:1773941319.284115:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ffda2b1e. 00000100:00000010:1.0:1773941319.284118:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000005ed71251. 00000100:00000010:1.0:1773941319.284122:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b5f44674. 00000100:00000010:1.0:1773941319.284125:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ee5342e6. 00000400:00000200:0.0:1773941319.284128:0:2350:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00000010:1.0:1773941319.284135:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000086c43c9c. 00000400:00000200:0.0:1773941319.284135:0:2350:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.143@tcp(00000000cbaeb348) state 0x1000c1 00000400:00000010:0.0:1773941319.284140:0:2350:0:(peer.c:3265:lnet_peer_merge_data()) alloc '(curnis)': 40 at 000000007b0f354c. 00000100:00000010:1.0:1773941319.284142:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000eb64ac7b. 00000400:00000010:0.0:1773941319.284144:0:2350:0:(peer.c:3266:lnet_peer_merge_data()) alloc '(addnis)': 48 at 000000009c017ad8. 00000400:00000010:0.0:1773941319.284147:0:2350:0:(peer.c:3267:lnet_peer_merge_data()) alloc '(delnis)': 40 at 000000007f0796df. 00000100:00000010:1.0:1773941319.284148:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000261a1d52. 00000400:00000200:0.0:1773941319.284152:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941319.284156:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000009a6cb6c0. 00000400:00000200:0.0:1773941319.284156:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941319.284159:0:2350:0:(peer.c:3405:lnet_peer_merge_data()) kfreed 'curnis': 40 at 000000007b0f354c. 00000400:00000010:0.0:1773941319.284162:0:2350:0:(peer.c:3406:lnet_peer_merge_data()) kfreed 'addnis': 48 at 000000009c017ad8. 00000100:00000010:1.0:1773941319.284163:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000fc09bf49. 00000400:00000010:0.0:1773941319.284163:0:2350:0:(peer.c:3407:lnet_peer_merge_data()) kfreed 'delnis': 40 at 000000007f0796df. 00000400:00000010:0.0:1773941319.284168:0:2350:0:(api-ni.c:1839:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 00000000a18edf7e. 00000100:00000010:1.0:1773941319.284170:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000934e9167. 00000400:00000200:0.0:1773941319.284171:0:2350:0:(peer.c:3410:lnet_peer_merge_data()) peer 192.168.204.143@tcp (00000000cbaeb348): 0 00000400:00000200:0.0:1773941319.284176:0:2350:0:(peer.c:3714:lnet_peer_data_present()) peer 192.168.204.143@tcp(00000000cbaeb348): 0. state = 0x100141 00000100:00000010:1.0:1773941319.284178:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000079b27c6f. 00000400:00000200:0.0:1773941319.284178:0:2350:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.143@tcp(00000000cbaeb348) state 0x100141 rc 1 00000400:00000200:0.0:1773941319.284181:0:2350:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.143@tcp(00000000cbaeb348) state 0x100141 00000100:00000010:1.0:1773941319.284184:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000214de9dc. 00000400:00000010:0.0:1773941319.284202:0:2350:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e2f9002c. 00000100:00000010:1.0:1773941319.284211:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d480b504. 00000400:00000010:0.0:1773941319.284218:0:2350:0:(lib-lnet.h:478:lnet_rspt_alloc()) alloc 'rspt': 56 at 00000000cab0d3bd. 00000400:00000200:0.0:1773941319.284223:0:2350:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000100:00000010:1.0:1773941319.284224:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000abd2ebe9. 00000400:00000200:0.0:1773941319.284228:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941319.284229:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000002c178be5. 00000400:00000200:0.0:1773941319.284233:0:2350:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source Specified: 192.168.204.43@tcp to MR: 192.168.204.143@tcp local destination 00000100:00000010:1.0:1773941319.284236:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000978d5369. 00000400:00000200:0.0:1773941319.284240:0:2350:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [2:2:256:4294967295] 192.168.204.143@tcp LPNI seq info [2:2:8:4294967295] 00000100:00000010:1.0:1773941319.284242:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000076ca2a2a. 00000400:00000200:0.0:1773941319.284244:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.284248:0:2350:0:(lib-move.c:2009:lnet_handle_send()) rspt_next_hop_nid = 192.168.204.143@tcp 00000100:00000010:1.0:1773941319.284249:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000004be6aa4. 00000400:00000200:0.0:1773941319.284254:0:2350:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:192.168.204.43@tcp) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000100:00000010:1.0:1773941319.284256:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000fcaf797d. 00000800:00000200:0.0:1773941319.284259:0:2350:0:(socklnd_cb.c:1001:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.204.143@tcp 00000100:00000010:1.0:1773941319.284262:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000002624ac76. 00000800:00000010:0.0:1773941319.284263:0:2350:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009357e6d0. 00000800:00000200:0.0:1773941319.284268:0:2350:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (3) 00000800:00000200:0.0:1773941319.284276:0:2350:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941319.284278:0:2350:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000009357e6d0 type 1, nob 144 niov 1 nkiov 1 00000100:00000010:1.0:1773941319.284283:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000076cbe6f. 00000400:00000200:0.0:1773941319.284288:0:2350:0:(peer.c:3922:lnet_peer_send_push()) peer 192.168.204.143@tcp 00000100:00000010:1.0:1773941319.284289:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000072b373e9. 00000400:00000200:0.0:1773941319.284293:0:2350:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.143@tcp(00000000cbaeb348) state 0x100541 rc 0 00000100:00000010:1.0:1773941319.284295:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000001a6cf19f. 00000100:00000010:1.0:1773941319.284300:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003a2549d1. 00000100:00000010:1.0:1773941319.284304:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000001512431a. 00000100:00000010:1.0:1773941319.284307:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000dd01718a. 00000100:00000010:1.0:1773941319.284311:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e1062d76. 00000100:00000010:1.0:1773941319.284315:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ea879e82. 00000100:00000010:1.0:1773941319.284325:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000c1834a2b. 00000100:00000010:1.0:1773941319.284329:0:5414:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at 0000000045d9966f. 00000100:00000010:1.0:1773941319.284339:0:5414:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at 00000000822168e7. 00000800:00000200:2.0:1773941319.284343:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941319.284346:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009357e6d0. 00000400:00000200:2.0:1773941319.284350:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000010:1.0:1773941319.284351:0:5414:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at 00000000170a6319. 00000400:00000200:2.0:1773941319.284357:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941319.284360:0:2347:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1773941319.284363:0:2347:0:(peer.c:3033:lnet_discovery_event_send()) Push Send to 192.168.204.143@tcp: 0 00000100:00000010:1.0:1773941319.284364:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000001994ebfe. 00000100:00000010:1.0:1773941319.284369:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e8dcb0a4. 00000100:00000010:1.0:1773941319.284372:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e2b360b4. 00000100:00000010:1.0:1773941319.284376:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e33556bc. 00000100:00000010:1.0:1773941319.284378:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000013ac39d7. 00000100:00000010:1.0:1773941319.284382:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b094d7f1. 00000100:00000010:1.0:1773941319.284383:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000007e9a275f. 00000100:00000010:1.0:1773941319.284392:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000004afc41f4. 00000100:00000010:1.0:1773941319.284395:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000001f5635f1. 00000100:00000010:1.0:1773941319.284398:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e36c84cf. 00000100:00000010:1.0:1773941319.284400:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c112a910. 00000100:00000010:1.0:1773941319.284404:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000074277733. 00000100:00000010:1.0:1773941319.284409:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b584535c. 00000100:00000010:1.0:1773941319.284413:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000fb476d27. 00000100:00000010:1.0:1773941319.284415:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f8b46850. 00000100:00000010:1.0:1773941319.284423:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000008d26234e. 00000100:00000010:1.0:1773941319.284425:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000366fb336. 00000100:00000010:1.0:1773941319.284438:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000bb9764f3. 00000100:00000010:1.0:1773941319.284442:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000040d9343a. 00000100:00000010:1.0:1773941319.284446:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000004728c2f7. 00000100:00000010:1.0:1773941319.284449:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000044db0bb3. 00000100:00000010:1.0:1773941319.284453:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d052a7f4. 00000100:00000010:1.0:1773941319.284457:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000002db2e796. 00000100:00000010:1.0:1773941319.284469:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000001b6c2732. 00000100:00000010:1.0:1773941319.284472:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000006e99172. 00000100:00000010:1.0:1773941319.284477:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000000ea11201. 00000100:00000010:1.0:1773941319.284479:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000888f4b9b. 00000100:00000010:1.0:1773941319.284483:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000dc64589a. 00000100:00000010:1.0:1773941319.284486:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000009acd8b4d. 00000100:00000010:1.0:1773941319.284489:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000065b4164a. 00000100:00000010:1.0:1773941319.284492:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000393132f9. 00000100:00000010:1.0:1773941319.284501:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000c1af2939. 00000100:00000010:1.0:1773941319.284510:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000a3a2d9c5. 00000100:00000010:1.0:1773941319.284516:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000080d8e3eb. 00000100:00000010:1.0:1773941319.284519:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000005145a1f7. 00000100:00000010:1.0:1773941319.284523:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000029d12c98. 00000100:00000010:1.0:1773941319.284526:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006e7d123e. 00000100:00000010:1.0:1773941319.284530:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000003d83522. 00000400:00000200:0.0:1773941319.284532:0:2339:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941319.284534:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000eeb44576. 00000800:00000200:0.1:1773941319.284538:0:2339:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:0.0:1773941319.284545:0:2339:0:(socklnd.c:1183:ksocknal_create_conn()) New conn 12345-192.168.204.143@tcp p 3.x 192.168.204.43 -> 192.168.204.143:988 incarnation:1773941276392337894 sched[1] 00000100:00000010:1.0:1773941319.284546:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b5d476d1. 00000800:00000010:0.0:1773941319.284550:0:2339:0:(socklnd.c:1191:ksocknal_create_conn()) kfreed 'hello': 144 at 000000009faddc4e. 00000100:00000010:1.0:1773941319.284552:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000d57585e2. 00000100:00000010:1.0:1773941319.284559:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000000c0a84f2. 00000100:00000010:1.0:1773941319.284562:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ce30cef7. 00000100:00000010:1.0:1773941319.284570:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d7cff103. 00000100:00000010:1.0:1773941319.284575:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000083ab4dca. 00000100:00000010:1.0:1773941319.284580:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000009bb71928. 00000100:00000010:1.0:1773941319.284584:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b61aef48. 00000100:00000010:1.0:1773941319.284593:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000325560be. 00000400:00000200:0.0:1773941319.284595:0:2339:0:(lib-socket.c:453:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.204.143:988 00000100:00000010:1.0:1773941319.284597:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000020e207ae. 00000100:00000010:1.0:1773941319.284603:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000df872f5d. 00000100:00000010:1.0:1773941319.284606:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000002934e7. 00000100:00000010:1.0:1773941319.284610:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ca6a3dc3. 00000100:00000010:1.0:1773941319.284613:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000018922b4d. 00000100:00000010:1.0:1773941319.284618:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000bc9d8ee8. 00000100:00000010:1.0:1773941319.284622:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000414b82ab. 00000400:00000200:0.0:1773941319.284625:0:2339:0:(lib-socket.c:453:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.204.143:988 00000100:00000010:1.0:1773941319.284632:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000f6905db2. 00000100:00000010:1.0:1773941319.284638:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000a3ae3cc3. 00000100:00000010:1.0:1773941319.284642:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000209fb999. 00000100:00000010:1.0:1773941319.284647:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000d29a9191. 00000100:00000010:1.0:1773941319.284652:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000709916f3. 00000100:00000010:1.0:1773941319.284656:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000043d29d1b. 00000100:00000010:1.0:1773941319.284660:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a7c8b0fb. 00000100:00000010:1.0:1773941319.284663:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000000c3b639d. 00000100:00000010:1.0:1773941319.284673:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000001734bd26. 00000100:00000010:1.0:1773941319.284683:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000bd062375. 00000100:00000010:1.0:1773941319.284686:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000822460cd. 00000100:00000010:1.0:1773941319.284689:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000004d071b53. 00000100:00000010:1.0:1773941319.284691:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000006f541afb. 00000100:00000010:1.0:1773941319.284693:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f8addd98. 00000100:00000010:1.0:1773941319.284696:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000da2b2a13. 00000100:00000010:1.0:1773941319.284698:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000d2bad114. 00000100:00000010:1.0:1773941319.284725:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e2dca8da. 00000400:00000200:2.0:1773941319.284729:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941319.284729:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e3e5e259. 00000100:00000010:1.0:1773941319.284734:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000079b57cc0. 00000400:00000200:2.0:1773941319.284735:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : ACK - for me 00000100:00000010:1.0:1773941319.284741:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000279fa9c4. 00000100:00000010:1.0:1773941319.284744:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d81772c3. 00000100:00000010:1.0:1773941319.284746:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b507db11. 00000400:00000200:2.0:1773941319.284750:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941319.284750:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000f8329666. 00000400:00000200:2.0:1773941319.284754:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941319.284756:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e9fb8466. 00000400:00000200:2.0:1773941319.284764:0:2347:0:(lib-move.c:4700:lnet_parse_ack()) 192.168.204.43@tcp: ACK from 12345-192.168.204.143@tcp into md 0x21 00000400:00000200:2.0:1773941319.284767:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000010:1.0:1773941319.284769:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000880f4d19. 00000400:00000200:2.0:1773941319.284770:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: ACK: OK 00000100:00000010:1.0:1773941319.284774:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000aa30b58a. 00000400:00000200:2.0:1773941319.284775:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e2f9002c 00000400:00000010:2.0:1773941319.284778:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000e2f9002c. 00000400:00000200:2.0:1773941319.284781:0:2347:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 4 00000100:00000010:1.0:1773941319.284781:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d31fcb5b. 00000400:00000200:2.0:1773941319.284785:0:2347:0:(peer.c:2799:lnet_discovery_event_ack()) peer 192.168.204.143@tcp ev->status 0 00000100:00000010:1.0:1773941319.284786:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000517f8b9b. 00000400:00000200:2.0:1773941319.284790:0:2347:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.143@tcp: -114 00000100:00000010:1.0:1773941319.284792:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a3ae664c. 00000100:00000010:1.0:1773941319.284796:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000002e9c1976. 00000100:00000010:1.0:1773941319.284800:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000007e313406. 00000100:00000010:1.0:1773941319.284803:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000030eaaeec. 00000400:00000200:0.0:1773941319.284810:0:2350:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000100:00000010:1.0:1773941319.284811:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000088b16c5e. 00000100:00000010:1.0:1773941319.284815:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006010ad71. 00000400:00000200:0.0:1773941319.284818:0:2350:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.143@tcp(00000000cbaeb348) state 0x100141 00000100:00000010:1.0:1773941319.284821:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d06841dc. 00000400:00000200:0.0:1773941319.284824:0:2350:0:(peer.c:3844:lnet_peer_discovered()) peer 192.168.204.143@tcp 00000100:00000010:1.0:1773941319.284827:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000bc412ed4. 00000400:00000200:0.0:1773941319.284828:0:2350:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.143@tcp(00000000cbaeb348) state 0x100111 rc 0 00000100:00000010:1.0:1773941319.284834:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000009e06aee3. 00000400:00000200:0.0:1773941319.284835:0:2350:0:(peer.c:2421:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.204.143@tcp 00000100:00000010:1.0:1773941319.284841:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ebb83a74. 00000100:00000010:1.0:1773941319.284846:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000da89978a. 00000100:00000010:1.0:1773941319.284849:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000a1d82a75. 00000100:00000010:1.0:1773941319.284859:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000663e88ea. 00000100:00000010:1.0:1773941319.284868:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000842200e6. 00000400:00000200:2.0:1773941319.284873:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941319.284874:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000019754993. 00000400:00000200:2.0:1773941319.284879:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000100:00000010:1.0:1773941319.284880:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000005e49276. 00000400:00000200:2.0:1773941319.284884:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.284886:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941319.284886:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000006a059dd2. 00000400:00000200:2.0:1773941319.284890:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941319.284890:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006001425f. 00000400:00000200:2.0:1773941319.284894:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 48 into portal 0 MB=0x8000000000000000 00000100:00000010:1.0:1773941319.284895:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ddb7112d. 00000100:00000010:1.0:1773941319.284898:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000087c0d61. 00000400:00000200:2.0:1773941319.284901:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 0 from 12345-192.168.204.143@tcp of length 48/48 into md 0x9 [1] + 0 00000400:00000010:2.0:1773941319.284904:0:2346:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000009a7ffedd. 00000400:00000200:2.0:1773941319.284907:0:2346:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000fe36797e 00000100:00000010:1.0:1773941319.284907:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000030c9b9ec. 00000100:00000010:1.0:1773941319.284911:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c689d5dc. 00000100:00000010:1.0:1773941319.284919:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000fbe54751. 00000100:00000010:1.0:1773941319.284922:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006bb6b2ac. 00000100:00000010:1.0:1773941319.284925:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ad852751. 00000100:00000010:1.0:1773941319.284927:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000057f107f4. 00000100:00000010:1.0:1773941319.284931:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000073f6ffc8. 00000400:00000200:2.0:1773941319.284932:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000010:1.0:1773941319.284933:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003ef38282. 00000400:00000200:2.0:1773941319.284941:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000010:1.0:1773941319.284942:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000001086b401. 00000400:00000200:2.0:1773941319.284944:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000fe36797e 00000100:00000010:1.0:1773941319.284945:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003ff8ab5a. 00000100:00000010:1.0:1773941319.284948:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000002c80dbd7. 00000100:00000010:1.0:1773941319.284951:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000fffe3776. 00000400:00000010:2.0:1773941319.284955:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000fe36797e. 00000100:00000010:1.0:1773941319.284956:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000cd23ba3b. 00000400:00000200:2.0:1773941319.284961:0:2346:0:(api-ni.c:2341:lnet_push_target_event_handler()) type 2 status 0 unlinked 1 00000100:00000010:1.0:1773941319.284961:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000d34039e3. 00000400:00000200:2.0:1773941319.284967:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941319.284967:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000006ce6775. 00000400:00000200:2.0:1773941319.284972:0:2346:0:(peer.c:2570:lnet_peer_push_event()) peer 192.168.204.143@tcp(00000000cbaeb348) is MR 00000100:00000010:1.0:1773941319.284972:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b26e4912. 00000400:00000010:2.0:1773941319.284977:0:2346:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at 00000000c6ad703b. 00000400:00000200:2.0:1773941319.284980:0:2346:0:(peer.c:2644:lnet_peer_push_event()) Received Push 192.168.204.143@tcp 363512030 00000100:00000010:1.0:1773941319.284982:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b204ac2d. 00000100:00000010:1.0:1773941319.284985:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000073980350. 00000400:00000200:2.0:1773941319.284988:0:2346:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.143@tcp: 0 00000100:00000010:1.0:1773941319.284989:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ed07ce66. 00000100:00000010:1.0:1773941319.284992:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000de3c8586. 00000400:00000200:2.0:1773941319.284993:0:2346:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:2.0:1773941319.284996:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941319.284997:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000063b47948. 00000400:00000200:2.0:1773941319.285001:0:2346:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source Specified: 192.168.204.43@tcp to MR: 192.168.204.143@tcp local destination 00000100:00000010:1.0:1773941319.285002:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b75cfc26. 00000400:00000200:2.0:1773941319.285005:0:2346:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [3:3:256:4294967295] 192.168.204.143@tcp LPNI seq info [3:3:8:4294967295] 00000100:00000010:1.0:1773941319.285006:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000007a160983. 00000800:00000010:0.0:1773941319.285007:0:2339:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4712 at 00000000c048a33c. 00000400:00000200:2.0:1773941319.285011:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941319.285011:0:5414:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000bae1e241. 00000800:00000010:0.0:1773941319.285014:0:2339:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 000000009faddc4e. 00000400:00000200:2.0:1773941319.285020:0:2346:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:192.168.204.43@tcp) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) 192.168.204.143@tcp : ACK try# 0 00000800:00000010:0.0:1773941319.285020:0:2339:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 000000007f0796df. 00000100:00000010:1.0:1773941319.285023:0:5414:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e3897c32. 00000800:00000200:2.0:1773941319.285026:0:2346:0:(socklnd_cb.c:1001:ksocknal_send()) sending 0 bytes in 1 frags to 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941319.285028:0:5414:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 17 lazy 00000800:00000010:2.0:1773941319.285030:0:2346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000bd41f1d3. 00000800:00000010:0.0:1773941319.285031:0:2339:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 000000007f0796df. 00000800:00000200:2.0:1773941319.285037:0:2346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (4) 00000800:00000200:2.0:1773941319.285042:0:2346:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941319.285044:0:2346:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000bd41f1d3 type 0, nob 96 niov 1 nkiov 0 00000400:00000200:0.0:1773941319.285049:0:2350:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000010:0.0:1773941319.285054:0:2350:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000088ad9eff. 00000400:00000010:0.0:1773941319.285060:0:2350:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000ec3181ff. 00000400:00000200:0.0:1773941319.285064:0:2350:0:(api-ni.c:2331:lnet_push_target_post()) posted push target 00000000ca5e5320 00000400:00000200:0.0:1773941319.285070:0:2350:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.143@tcp(00000000cbaeb348) state 0x1000d1 00000400:00000010:0.0:1773941319.285075:0:2350:0:(peer.c:3265:lnet_peer_merge_data()) alloc '(curnis)': 40 at 000000007f0796df. 00000020:00000010:1.0:1773941319.285079:0:5414:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 000000007d6e9f61. 00000400:00000010:0.0:1773941319.285079:0:2350:0:(peer.c:3266:lnet_peer_merge_data()) alloc '(addnis)': 48 at 000000009c017ad8. 00000800:00000200:2.0:1773941319.285081:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941319.285085:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000bd41f1d3. 00000020:00000010:1.0:1773941319.285086:0:5414:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 0000000077bf842a. 00000400:00000010:0.0:1773941319.285087:0:2350:0:(peer.c:3267:lnet_peer_merge_data()) alloc '(delnis)': 40 at 000000007b0f354c. 00000400:00000200:2.0:1773941319.285090:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941319.285094:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.285098:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: ACK: OK 00000400:00000200:0.0:1773941319.285102:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941319.285106:0:2350:0:(peer.c:3405:lnet_peer_merge_data()) kfreed 'curnis': 40 at 000000007f0796df. 00000400:00000010:0.0:1773941319.285110:0:2350:0:(peer.c:3406:lnet_peer_merge_data()) kfreed 'addnis': 48 at 000000009c017ad8. 00000400:00000010:0.0:1773941319.285112:0:2350:0:(peer.c:3407:lnet_peer_merge_data()) kfreed 'delnis': 40 at 000000007b0f354c. 00000400:00000010:0.0:1773941319.285116:0:2350:0:(api-ni.c:1839:lnet_ping_buffer_free()) kfreed 'pbuf': 57 at 00000000c6ad703b. 00000100:00000010:1.0:1773941319.285125:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000001495a169. 00000400:00000200:0.0:1773941319.285127:0:2350:0:(peer.c:3410:lnet_peer_merge_data()) peer 192.168.204.143@tcp (00000000cbaeb348): 0 00000100:00000010:1.0:1773941319.285130:0:5414:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 000000002a76e403. 00000400:00000200:0.0:1773941319.285133:0:2350:0:(peer.c:3714:lnet_peer_data_present()) peer 192.168.204.143@tcp(00000000cbaeb348): 0. state = 0x100151 00000100:00000010:1.0:1773941319.285134:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000005f6e1613. 00000100:00000010:1.0:1773941319.285139:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000005e540474. 00000400:00000200:0.0:1773941319.285141:0:2350:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.143@tcp(00000000cbaeb348) state 0x100151 rc 1 00000100:00000010:1.0:1773941319.285143:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000003b7a85dc. 00000100:00000010:1.0:1773941319.285145:0:5414:0:(nrs.c:1026:nrs_svcpt_setup_locked()) kmalloced '(svcpt->scp_nrs_hp)': 112 at 00000000a3225626. 00000100:00000010:1.0:1773941319.285153:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000e11dc949. 00000100:00000010:1.0:1773941319.285156:0:5414:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 0000000076c7c697. 00000400:00000200:0.0:1773941319.285156:0:2350:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.143@tcp(00000000cbaeb348) state 0x100151 00000100:00000010:1.0:1773941319.285160:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000068737eef. 00000400:00000200:0.0:1773941319.285163:0:2350:0:(peer.c:3844:lnet_peer_discovered()) peer 192.168.204.143@tcp 00000100:00000010:1.0:1773941319.285164:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000002c052709. 00000100:00000010:1.0:1773941319.285168:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000072854dc6. 00000400:00000200:0.0:1773941319.285168:0:2350:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.143@tcp(00000000cbaeb348) state 0x100111 rc 0 00000100:00000010:1.0:1773941319.285170:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000c8237319. 00000100:00000010:1.0:1773941319.285174:0:5414:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 000000006232183d. 00000400:00000200:0.0:1773941319.285175:0:2350:0:(peer.c:2421:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.204.143@tcp 00000100:00000010:1.0:1773941319.285176:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000000e0df9fa. 00000100:00000010:1.0:1773941319.285179:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000d91ba57c. 00000100:00000010:1.0:1773941319.285181:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000a28b4561. 00000100:00000010:1.0:1773941319.285183:0:5414:0:(nrs.c:1026:nrs_svcpt_setup_locked()) kmalloced '(svcpt->scp_nrs_hp)': 112 at 00000000d4b5f1ad. 00000100:00000010:1.0:1773941319.285184:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000c9fd77e8. 00000100:00000010:1.0:1773941319.285185:0:5414:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 000000001ed00bb6. 00000100:00000010:1.0:1773941319.285186:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000008e7d730e. 00000100:00000010:1.0:1773941319.285187:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000029517d76. 00000100:00000010:1.0:1773941319.285206:0:5414:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000f44acfac. 00000100:00000200:1.0:1773941319.285227:0:5414:0:(service.c:848:ptlrpc_register_service()) ldlm_canceld: Started, listening on portal 17 00000100:00000010:1.0:1773941319.285229:0:5414:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 000000008a0e8d19. 00000100:00000010:1.0:1773941319.285458:0:5419:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 00000000ecb905ee. 00000020:00000010:1.0:1773941319.285462:0:5419:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000076cc5af. 00000020:00000010:1.0:1773941319.285466:0:5419:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006401340e. 00000020:00000010:1.0:1773941319.285468:0:5419:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000088822b17. 00000020:00000010:1.0:1773941319.285471:0:5419:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 0000000092e1a746. 00000040:00000010:1.0:1773941319.285474:0:5419:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000002bbe7aac. 00000020:00000010:1.0:1773941319.285477:0:5419:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000df5b0300. 00000020:00000010:1.0:1773941319.285480:0:5419:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 000000006be67b87. 00000020:00000010:1.0:1773941319.285483:0:5419:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000058a29e9. 80000000:00000010:1.0:1773941319.285485:0:5419:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000acdaed60. 40000000:00000010:1.0:1773941319.285487:0:5419:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000749b26b8. 00000008:00000010:1.0:1773941319.285499:0:5419:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000792a45b7. 00020000:00000010:1.0:1773941319.285502:0:5419:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 0000000034e5cb6d. 00008000:00000010:1.0:1773941319.285510:0:5419:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000005a29b78a. 00100000:00000010:1.0:1773941319.285513:0:5419:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000014ca31c. 00040000:00000010:1.0:1773941319.285531:0:5419:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000003d12aeb3. 00040000:00000010:1.0:1773941319.285534:0:5419:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000000e4653eb. 00040000:00000010:1.0:1773941319.285537:0:5419:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000fd2a1553. 00000004:00000010:1.0:1773941319.285540:0:5419:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000f7cc1942. 00000004:00000010:1.0:1773941319.285543:0:5419:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 00000000364cd083. 00000004:00000010:1.0:1773941319.285546:0:5419:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 0000000028e4b278. 00002000:00000010:1.0:1773941319.285548:0:5419:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 000000003660eb8c. 00000004:00000010:1.0:1773941319.285551:0:5419:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 000000008a22b366. 00000080:00000010:1.0:1773941319.285562:0:5419:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000006676772a. 00000080:00000010:1.0:1773941319.285579:0:5419:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 0000000098721126. 00000100:00000200:1.0:1773941319.285584:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285588:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000067bfa45f. 00000400:00000010:1.0:1773941319.285592:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c7cb6346. 00000100:00000200:1.0:1773941319.285595:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285597:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c7547fb7. 00000400:00000010:1.0:1773941319.285600:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000002bf094f6. 00000100:00000200:1.0:1773941319.285602:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285604:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000032e3208e. 00000400:00000010:1.0:1773941319.285606:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007d1662fe. 00000100:00000200:1.0:1773941319.285609:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285611:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000252864ba. 00000400:00000010:1.0:1773941319.285614:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c0c5a81d. 00000100:00000200:1.0:1773941319.285616:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285618:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000947ee5f0. 00000400:00000010:1.0:1773941319.285620:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000078f87a08. 00000100:00000200:1.0:1773941319.285622:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285624:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009b07e600. 00000400:00000010:1.0:1773941319.285626:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003344824a. 00000100:00000200:1.0:1773941319.285627:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285629:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004b7ded54. 00000400:00000010:1.0:1773941319.285631:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000836662b3. 00000100:00000200:1.0:1773941319.285633:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285635:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000384595e9. 00000400:00000010:1.0:1773941319.285638:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f0604b7b. 00000100:00000200:1.0:1773941319.285640:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285648:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000002704a4ff. 00000400:00000010:1.0:1773941319.285651:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d8e54da4. 00000100:00000200:1.0:1773941319.285653:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285655:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000088f9c551. 00000400:00000010:1.0:1773941319.285657:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000005c77b76. 00000100:00000200:1.0:1773941319.285659:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285661:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000006b325770. 00000400:00000010:1.0:1773941319.285663:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000bcf56c6d. 00000100:00000200:1.0:1773941319.285665:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285667:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e6664167. 00000400:00000010:1.0:1773941319.285669:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003a9dbc65. 00000100:00000200:1.0:1773941319.285670:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285672:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003a47d956. 00000400:00000010:1.0:1773941319.285674:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003dd556cb. 00000100:00000200:1.0:1773941319.285676:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285678:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000d129a42. 00000400:00000010:1.0:1773941319.285681:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008ada6ab8. 00000100:00000200:1.0:1773941319.285684:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285686:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e5165c52. 00000400:00000010:1.0:1773941319.285690:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008d58479b. 00000100:00000200:1.0:1773941319.285692:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285695:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000002bfa6839. 00000400:00000200:0.0:1773941319.285697:0:2339:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941319.285700:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d4a5680c. 00000800:00000200:0.1:1773941319.285702:0:2339:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected e ksnr_max_conns 1 00000800:00000200:0.0:1773941319.285728:0:2339:0:(socklnd.c:1183:ksocknal_create_conn()) New conn 12345-192.168.204.143@tcp p 3.x 192.168.204.43 -> 192.168.204.143:988 incarnation:1773941276392337894 sched[1] 00000100:00000200:1.0:1773941319.285731:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000800:00000010:0.0:1773941319.285734:0:2339:0:(socklnd.c:1191:ksocknal_create_conn()) kfreed 'hello': 144 at 000000009faddc4e. 00000400:00000010:1.0:1773941319.285736:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003535fb97. 00000400:00000010:1.0:1773941319.285741:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f8d27738. 00000100:00000200:1.0:1773941319.285744:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285746:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000021eb7ab2. 00000400:00000010:1.0:1773941319.285749:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000984c50db. 00000100:00000200:1.0:1773941319.285752:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285754:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000001992b587. 00000400:00000010:1.0:1773941319.285757:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000016a5fc8c. 00000100:00000200:1.0:1773941319.285759:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285761:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005ff0a844. 00000400:00000010:1.0:1773941319.285764:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d6c58581. 00000100:00000200:1.0:1773941319.285772:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285775:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ddd97589. 00000400:00000010:1.0:1773941319.285778:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000daf166ec. 00000100:00000200:1.0:1773941319.285781:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285783:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000020a83c7c. 00000400:00000010:1.0:1773941319.285786:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000002590abc4. 00000100:00000200:1.0:1773941319.285788:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285789:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003bebf392. 00000400:00000010:1.0:1773941319.285791:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f8437d15. 00000100:00000200:1.0:1773941319.285793:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285794:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000da49dff4. 00000400:00000010:1.0:1773941319.285796:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000059f7a529. 00000100:00000200:1.0:1773941319.285797:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285799:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ca3c589b. 00000400:00000010:1.0:1773941319.285801:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ed0b59c5. 00000100:00000200:1.0:1773941319.285803:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285804:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000008be3a05. 00000400:00000010:1.0:1773941319.285807:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c65f2080. 00000100:00000200:1.0:1773941319.285808:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285810:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000089aec949. 00000400:00000010:1.0:1773941319.285812:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e6808f88. 00000100:00000200:1.0:1773941319.285814:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285816:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b3fd939a. 00000400:00000010:1.0:1773941319.285817:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ca5d0809. 00000100:00000200:1.0:1773941319.285819:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285821:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007df71206. 00000400:00000010:1.0:1773941319.285823:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000897629d4. 00000100:00000200:1.0:1773941319.285825:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285826:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c4ef7573. 00000400:00000010:1.0:1773941319.285828:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003b2c8b4c. 00000100:00000200:1.0:1773941319.285830:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285831:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a4292ddc. 00000400:00000010:1.0:1773941319.285833:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008d5856b0. 00000100:00000200:1.0:1773941319.285835:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285837:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000004db8e85. 00000400:00000010:1.0:1773941319.285844:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007f33fec5. 00000100:00000200:1.0:1773941319.285846:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285848:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000008c80c42d. 00000400:00000010:1.0:1773941319.285850:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000029fe090c. 00000100:00000200:1.0:1773941319.285851:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285853:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000cf15a710. 00000400:00000010:1.0:1773941319.285855:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001a74c399. 00000100:00000200:1.0:1773941319.285856:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285858:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a8ecc9b4. 00000400:00000010:1.0:1773941319.285860:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000034a05d52. 00000100:00000200:1.0:1773941319.285862:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285863:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000001423c967. 00000400:00000010:1.0:1773941319.285873:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005204b609. 00000100:00000200:1.0:1773941319.285877:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285879:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005a67f01d. 00000400:00000010:1.0:1773941319.285881:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d1048b12. 00000100:00000200:1.0:1773941319.285883:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285885:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000044ebdc25. 00000400:00000010:1.0:1773941319.285887:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000028ba974b. 00000100:00000200:1.0:1773941319.285888:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285890:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000259a69d5. 00000400:00000010:1.0:1773941319.285892:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003c431ba3. 00000100:00000200:1.0:1773941319.285894:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285895:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ba1a062d. 00000400:00000010:1.0:1773941319.285897:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e78b2ce3. 00000100:00000200:1.0:1773941319.285899:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285900:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fd16c894. 00000400:00000010:1.0:1773941319.285902:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000adbfb9b5. 00000100:00000200:1.0:1773941319.285904:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285915:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000554adab6. 00000400:00000010:1.0:1773941319.285920:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005ce8b9a4. 00000100:00000200:1.0:1773941319.285922:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285924:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000068b697df. 00000400:00000010:1.0:1773941319.285926:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000000589a265. 00000100:00000200:1.0:1773941319.285928:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285934:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003de0e0e0. 00000400:00000010:1.0:1773941319.285936:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000181204a4. 00000100:00000200:1.0:1773941319.285938:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285940:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009a7ffedd. 00000400:00000010:1.0:1773941319.285941:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000bac61cc7. 00000100:00000200:1.0:1773941319.285943:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285951:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d7b802b4. 00000400:00000010:1.0:1773941319.285954:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ee729750. 00000100:00000200:1.0:1773941319.285957:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285959:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000047bd6548. 00000400:00000010:1.0:1773941319.285960:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000009c3b328c. 00000100:00000200:1.0:1773941319.285962:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285964:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000126c4df9. 00000400:00000010:1.0:1773941319.285966:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a6203532. 00000100:00000200:1.0:1773941319.285968:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285970:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000579a1e2b. 00000400:00000010:1.0:1773941319.285972:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000301114c3. 00000100:00000200:1.0:1773941319.285974:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285975:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000437abd31. 00000400:00000010:1.0:1773941319.285977:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000bfc1804e. 00000100:00000200:1.0:1773941319.285979:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285981:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000006525c7e9. 00000400:00000010:1.0:1773941319.285983:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000885de629. 00000100:00000200:1.0:1773941319.285985:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285986:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000069d1ee85. 00000400:00000010:1.0:1773941319.285988:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008a6e8367. 00000100:00000200:1.0:1773941319.285990:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285992:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000084ad317. 00000400:00000010:1.0:1773941319.285993:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000014ae71bb. 00000100:00000200:1.0:1773941319.285995:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.285997:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000091ad8c22. 00000400:00000010:1.0:1773941319.285999:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c50d7843. 00000100:00000200:1.0:1773941319.286000:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.286002:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000fd074de. 00000400:00000010:1.0:1773941319.286004:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000046880ffc. 00000100:00000200:1.0:1773941319.286010:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.286012:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003613fed0. 00000400:00000010:1.0:1773941319.286014:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007e99edb0. 00000100:00000200:1.0:1773941319.286016:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.286018:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000008eb3cf81. 00000400:00000010:1.0:1773941319.286026:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000043ab97c2. 00000100:00000200:1.0:1773941319.286029:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.286031:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000004eb5961. 00000400:00000010:1.0:1773941319.286033:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000000c8a1350. 00000100:00000200:1.0:1773941319.286035:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.286036:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000080d512bb. 00000400:00000010:1.0:1773941319.286038:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007e7a4c7b. 00000100:00000200:1.0:1773941319.286039:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.286041:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000260a1e49. 00000400:00000010:1.0:1773941319.286042:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e23b7315. 00000100:00000200:1.0:1773941319.286044:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.286045:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005fc44425. 00000400:00000010:1.0:1773941319.286047:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003dba0b83. 00000100:00000200:1.0:1773941319.286048:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.286050:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000006021d517. 00000400:00000010:1.0:1773941319.286051:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000036840ddc. 00000100:00000200:1.0:1773941319.286053:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.286054:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a4c82e94. 00000400:00000010:1.0:1773941319.286056:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ddda2843. 00000100:00000200:1.0:1773941319.286058:0:5419:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941319.286059:0:5419:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000be3cc5e9. 00000400:00000010:1.0:1773941319.286060:0:5419:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003145fa7f. 00000100:00000010:1.0:1773941319.286063:0:5419:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 0000000093ab7929. 00000100:00000200:1.0:1773941319.286075:0:5419:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:0.0:1773941319.286096:0:5414:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 00000000b483cde4. 00000100:00000010:0.0:1773941319.286258:0:5420:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 000000009c017ad8. 00000020:00000010:0.0:1773941319.286262:0:5420:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000007f0796df. 00000020:00000010:0.0:1773941319.286266:0:5420:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ae167c28. 00000020:00000010:0.0:1773941319.286268:0:5420:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000074e9fc85. 00000020:00000010:0.0:1773941319.286270:0:5420:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000fe531d08. 00000040:00000010:0.0:1773941319.286273:0:5420:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000c98f1b76. 00000020:00000010:0.0:1773941319.286275:0:5420:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000015e5a334. 00000020:00000010:0.0:1773941319.286279:0:5420:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000be25b41d. 00000020:00000010:0.0:1773941319.286281:0:5420:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000004b28cb70. 80000000:00000010:0.0:1773941319.286284:0:5420:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000b32a0089. 40000000:00000010:0.0:1773941319.286286:0:5420:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000c12a6ea8. 00000008:00000010:0.0:1773941319.286289:0:5420:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000f6ac0dc9. 00020000:00000010:0.0:1773941319.286292:0:5420:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000e827106f. 00008000:00000010:0.0:1773941319.286294:0:5420:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000006049aaed. 00100000:00000010:0.0:1773941319.286298:0:5420:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000d0ad335b. 00040000:00000010:0.0:1773941319.286300:0:5420:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000009ef0f798. 00040000:00000010:0.0:1773941319.286303:0:5420:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000008fc2f9b5. 00040000:00000010:0.0:1773941319.286305:0:5420:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000d5294313. 00000004:00000010:0.0:1773941319.286307:0:5420:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000002ee3cf84. 00000004:00000010:0.0:1773941319.286311:0:5420:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 00000000154afdc3. 00000004:00000010:0.0:1773941319.286314:0:5420:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 000000003d943dff. 00002000:00000010:0.0:1773941319.286321:0:5420:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000d3af0ef2. 00000004:00000010:0.0:1773941319.286324:0:5420:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 0000000011d01299. 00000080:00000010:0.0:1773941319.286327:0:5420:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000704ae6b2. 00000080:00000010:0.0:1773941319.286329:0:5420:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000bd13a37d. 00000100:00000010:0.0:1773941319.286333:0:5420:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 0000000095153a5b. 00000100:00000200:0.0:1773941319.286344:0:5420:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:1.0:1773941319.286364:0:5414:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 000000007961ecab. 00000100:00000010:1.0:1773941319.286517:0:5421:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 00000000c7d17efe. 00000020:00000010:1.0:1773941319.286535:0:5421:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000dc127b0d. 00000020:00000010:1.0:1773941319.286539:0:5421:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a18edf7e. 00000020:00000010:1.0:1773941319.286542:0:5421:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000cf9ff504. 00000020:00000010:1.0:1773941319.286545:0:5421:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 000000001c5c2370. 00000040:00000010:1.0:1773941319.286552:0:5421:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000003b95c363. 00000020:00000010:1.0:1773941319.286555:0:5421:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000001f1dd19c. 00000020:00000010:1.0:1773941319.286558:0:5421:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 0000000014ade96b. 00000020:00000010:1.0:1773941319.286561:0:5421:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000001c7a70b2. 80000000:00000010:1.0:1773941319.286563:0:5421:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000c1e49e81. 40000000:00000010:1.0:1773941319.286565:0:5421:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000ebb33b4b. 00000008:00000010:1.0:1773941319.286569:0:5421:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 0000000038b0d4cc. 00020000:00000010:1.0:1773941319.286572:0:5421:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000cf025e55. 00008000:00000010:1.0:1773941319.286574:0:5421:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 0000000082912ec8. 00100000:00000010:1.0:1773941319.286577:0:5421:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000cd8495bb. 00040000:00000010:1.0:1773941319.286580:0:5421:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000009588d892. 00040000:00000010:1.0:1773941319.286583:0:5421:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000dc4d591c. 00040000:00000010:1.0:1773941319.286585:0:5421:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000913df4ce. 00000004:00000010:1.0:1773941319.286588:0:5421:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000002bb0768f. 00000004:00000010:1.0:1773941319.286591:0:5421:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 000000002f45cfce. 00000004:00000010:1.0:1773941319.286595:0:5421:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 0000000077cbc780. 00002000:00000010:1.0:1773941319.286597:0:5421:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000a1b4c6e1. 00000004:00000010:1.0:1773941319.286600:0:5421:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 000000006c42aa86. 00000080:00000010:1.0:1773941319.286602:0:5421:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000082d490e. 00000080:00000010:1.0:1773941319.286604:0:5421:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 0000000056765afa. 00000100:00000010:1.0:1773941319.286616:0:5421:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 0000000014b1ede1. 00000100:00000200:1.0:1773941319.286629:0:5421:0:(service.c:3086:ptlrpc_main()) service thread 2 (#3) started 00000100:00000010:0.0:1773941319.286645:0:5414:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 0000000035ecc482. 00000100:00000010:2.0:1773941319.286823:0:5422:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 0000000035b69fb5. 00000020:00000010:2.0:1773941319.286826:0:5422:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 0000000033abec43. 00000020:00000010:2.0:1773941319.286830:0:5422:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005e501683. 00000020:00000010:2.0:1773941319.286833:0:5422:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000a4172525. 00000020:00000010:2.0:1773941319.286836:0:5422:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 0000000086e922f5. 00000040:00000010:2.0:1773941319.286838:0:5422:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000006fd6ff85. 00000020:00000010:2.0:1773941319.286841:0:5422:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000002e29664f. 00000020:00000010:2.0:1773941319.286844:0:5422:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 000000008974aca9. 00000020:00000010:2.0:1773941319.286847:0:5422:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000a54abd82. 80000000:00000010:2.0:1773941319.286854:0:5422:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 0000000083a94faa. 40000000:00000010:2.0:1773941319.286856:0:5422:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 0000000094752940. 00000008:00000010:2.0:1773941319.286866:0:5422:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000b8b1792a. 00020000:00000010:2.0:1773941319.286870:0:5422:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000004b449598. 00008000:00000010:2.0:1773941319.286878:0:5422:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000008271dde6. 00100000:00000010:2.0:1773941319.286882:0:5422:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 0000000027435f74. 00040000:00000010:2.0:1773941319.286884:0:5422:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000bd41f1d3. 00040000:00000010:2.0:1773941319.286887:0:5422:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000e4d95b9d. 00040000:00000010:2.0:1773941319.286890:0:5422:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000077f14e9a. 00000004:00000010:2.0:1773941319.286892:0:5422:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000eaddb2a1. 00000004:00000010:2.0:1773941319.286896:0:5422:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 0000000020a8f5a5. 00000004:00000010:2.0:1773941319.286899:0:5422:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 000000007dc9c778. 00002000:00000010:2.0:1773941319.286902:0:5422:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000e79184e2. 00000004:00000010:2.0:1773941319.286904:0:5422:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 00000000c8d3391e. 00000080:00000010:2.0:1773941319.286916:0:5422:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000006a848d3d. 00000080:00000010:2.0:1773941319.286927:0:5422:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000dcb29a68. 00000100:00000200:2.0:1773941319.286930:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.286934:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007636a3d5. 00000400:00000010:2.0:1773941319.286938:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000bb2d0ab0. 00000100:00000200:2.0:1773941319.286942:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.286944:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000061a2212e. 00000400:00000010:2.0:1773941319.286946:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e9968224. 00000100:00000200:2.0:1773941319.286948:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.286950:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000079db1431. 00000400:00000010:2.0:1773941319.286951:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c95b5f12. 00000100:00000200:2.0:1773941319.286953:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.286955:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f75dfc43. 00000400:00000010:2.0:1773941319.286957:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b14f4ae3. 00000100:00000200:2.0:1773941319.286959:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.286960:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000813a0fea. 00000400:00000010:2.0:1773941319.286962:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f6cfc120. 00000100:00000200:2.0:1773941319.286964:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.286966:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000002e3cf1bd. 00000400:00000010:2.0:1773941319.286968:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000556c86de. 00000100:00000200:2.0:1773941319.286975:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.286977:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000008031daa1. 00000400:00000010:2.0:1773941319.286979:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000002e492251. 00000100:00000200:2.0:1773941319.286980:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.286982:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000002b649c15. 00000400:00000010:2.0:1773941319.286984:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003e25e7c2. 00000100:00000200:2.0:1773941319.286986:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.286989:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000097476b54. 00000400:00000010:2.0:1773941319.286990:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000048328b32. 00000100:00000200:2.0:1773941319.286992:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.286994:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007123d125. 00000400:00000010:2.0:1773941319.286996:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000000ee94fed. 00000100:00000200:2.0:1773941319.286997:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.286999:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000040a4809. 00000400:00000010:2.0:1773941319.287001:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000043b5e16f. 00000100:00000200:2.0:1773941319.287003:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287005:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f43ec764. 00000400:00000010:2.0:1773941319.287007:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000027670e55. 00000100:00000200:2.0:1773941319.287009:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287011:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005b9bb743. 00000400:00000010:2.0:1773941319.287013:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000028374add. 00000100:00000200:2.0:1773941319.287015:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287017:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007ad3f419. 00000400:00000010:2.0:1773941319.287019:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000046c8dec6. 00000100:00000200:2.0:1773941319.287021:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287023:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009c34f7c8. 00000400:00000010:2.0:1773941319.287025:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f7d8b79f. 00000100:00000200:2.0:1773941319.287027:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287029:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f7cfbdca. 00000400:00000010:2.0:1773941319.287031:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007b144d7a. 00000100:00000200:2.0:1773941319.287033:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287034:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000080d58c6. 00000400:00000010:2.0:1773941319.287037:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000edf00827. 00000100:00000200:2.0:1773941319.287039:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287040:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004f34ac3e. 00000400:00000010:2.0:1773941319.287048:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000cc92a341. 00000100:00000200:2.0:1773941319.287050:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287052:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000245ba254. 00000400:00000010:2.0:1773941319.287054:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004be270fe. 00000100:00000200:2.0:1773941319.287056:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287065:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b2d328a1. 00000400:00000010:2.0:1773941319.287068:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000905b2053. 00000100:00000200:2.0:1773941319.287070:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287072:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ce14dc09. 00000400:00000010:2.0:1773941319.287074:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000977aa856. 00000100:00000200:2.0:1773941319.287076:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287077:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000921906fd. 00000400:00000010:2.0:1773941319.287079:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000093b1d555. 00000100:00000200:2.0:1773941319.287081:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287083:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000438c015a. 00000400:00000010:2.0:1773941319.287085:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000009637501e. 00000100:00000200:2.0:1773941319.287087:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287088:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000014ca785c. 00000400:00000010:2.0:1773941319.287090:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001b38fe1f. 00000100:00000200:2.0:1773941319.287092:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287093:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007956d2bf. 00000400:00000010:2.0:1773941319.287096:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e91d4f70. 00000100:00000200:2.0:1773941319.287098:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287099:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f7c35b18. 00000400:00000010:2.0:1773941319.287102:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000006f708114. 00000100:00000200:2.0:1773941319.287103:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287105:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d9ca7bc4. 00000400:00000010:2.0:1773941319.287107:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000031cf1bb3. 00000100:00000200:2.0:1773941319.287109:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287110:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003dd0a679. 00000400:00000010:2.0:1773941319.287112:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000029df40e7. 00000100:00000200:2.0:1773941319.287114:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287115:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003b691630. 00000400:00000010:2.0:1773941319.287117:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d8ea5a1a. 00000100:00000200:2.0:1773941319.287119:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287126:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000008014d510. 00000400:00000010:2.0:1773941319.287129:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d92f1fed. 00000100:00000200:2.0:1773941319.287132:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287134:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000032757cfe. 00000400:00000010:2.0:1773941319.287136:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003551cc7e. 00000100:00000200:2.0:1773941319.287137:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287139:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000253e38ea. 00000400:00000010:2.0:1773941319.287141:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b94dedc5. 00000100:00000200:2.0:1773941319.287143:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287144:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000099898160. 00000400:00000010:2.0:1773941319.287146:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008784e395. 00000100:00000200:2.0:1773941319.287148:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287149:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f06bc8ef. 00000400:00000010:2.0:1773941319.287152:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001410a5f0. 00000100:00000200:2.0:1773941319.287153:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287155:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000928bbf43. 00000400:00000010:2.0:1773941319.287157:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000119af45a. 00000100:00000200:2.0:1773941319.287158:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287160:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000b9c72b3. 00000400:00000010:2.0:1773941319.287162:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001e181ae3. 00000100:00000200:2.0:1773941319.287163:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287165:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000948b9798. 00000400:00000010:2.0:1773941319.287167:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000845e3aa4. 00000100:00000200:2.0:1773941319.287168:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287170:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000008d2a6ed9. 00000400:00000010:2.0:1773941319.287172:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000017ab39d8. 00000100:00000200:2.0:1773941319.287174:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287175:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009a044271. 00000400:00000010:2.0:1773941319.287177:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003fbf3018. 00000100:00000200:2.0:1773941319.287179:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287180:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c23f258b. 00000400:00000010:2.0:1773941319.287182:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001b6df4d3. 00000100:00000200:2.0:1773941319.287184:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287185:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000002524a9b2. 00000400:00000010:2.0:1773941319.287187:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d3ec9d06. 00000100:00000200:2.0:1773941319.287209:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287211:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000828461cd. 00000400:00000010:2.0:1773941319.287213:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008bc1a679. 00000100:00000200:2.0:1773941319.287215:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287217:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fe012075. 00000400:00000010:2.0:1773941319.287219:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000000493fa35. 00000100:00000200:2.0:1773941319.287221:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287222:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000068f1e112. 00000400:00000010:2.0:1773941319.287224:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000701dfe57. 00000100:00000200:2.0:1773941319.287226:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287228:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c948a290. 00000400:00000010:2.0:1773941319.287230:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000002e31b7fc. 00000100:00000200:2.0:1773941319.287231:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287233:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007c273856. 00000400:00000010:2.0:1773941319.287235:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c60e7042. 00000100:00000200:2.0:1773941319.287237:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287238:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000737243f0. 00000400:00000010:2.0:1773941319.287240:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000064c4f878. 00000100:00000200:2.0:1773941319.287242:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287244:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e9ed8d4e. 00000400:00000010:2.0:1773941319.287246:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008d5391c0. 00000100:00000200:2.0:1773941319.287248:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287250:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d3f65c36. 00000400:00000010:2.0:1773941319.287252:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ef862e6b. 00000100:00000200:2.0:1773941319.287253:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287255:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b937607a. 00000400:00000010:2.0:1773941319.287257:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000075a15583. 00000100:00000200:2.0:1773941319.287258:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287260:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000066256115. 00000400:00000010:2.0:1773941319.287262:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000269320be. 00000100:00000200:2.0:1773941319.287264:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287265:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000760ba586. 00000400:00000010:2.0:1773941319.287267:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007bb702eb. 00000100:00000200:2.0:1773941319.287269:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287271:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009c12ba6d. 00000400:00000010:2.0:1773941319.287277:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d93aeb1a. 00000100:00000200:2.0:1773941319.287280:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287282:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005b0513a2. 00000400:00000010:2.0:1773941319.287284:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008f36b580. 00000100:00000200:2.0:1773941319.287285:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287287:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000009d8e68c. 00000400:00000010:2.0:1773941319.287290:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004fb8d483. 00000100:00000200:2.0:1773941319.287291:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287293:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000cf2c3030. 00000400:00000010:2.0:1773941319.287295:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000032a17b6e. 00000100:00000200:2.0:1773941319.287297:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287298:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000008616fc9. 00000400:00000010:2.0:1773941319.287301:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ba254427. 00000100:00000200:2.0:1773941319.287303:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287305:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000001a1d47b4. 00000400:00000010:2.0:1773941319.287307:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000baa5dcd0. 00000100:00000200:2.0:1773941319.287309:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287311:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b8221569. 00000400:00000010:2.0:1773941319.287313:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b9d18752. 00000100:00000200:2.0:1773941319.287315:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287317:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000008124209b. 00000400:00000010:2.0:1773941319.287319:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003aaeb72f. 00000100:00000200:2.0:1773941319.287321:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287323:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000084d21ae1. 00000400:00000010:2.0:1773941319.287333:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000857ce216. 00000100:00000200:2.0:1773941319.287337:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287338:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000cab0cc4d. 00000400:00000010:2.0:1773941319.287340:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000455de67b. 00000100:00000200:2.0:1773941319.287342:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287344:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000081f6666f. 00000400:00000010:2.0:1773941319.287345:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d9bc10b1. 00000100:00000200:2.0:1773941319.287347:0:5422:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941319.287349:0:5422:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000f39c655. 00000400:00000010:2.0:1773941319.287350:0:5422:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000002482703a. 00000100:00000010:2.0:1773941319.287353:0:5422:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 0000000053260bbc. 00000100:00000200:2.0:1773941319.287376:0:5422:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:1.0:1773941319.287392:0:5414:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 00000000a99d80c6. 00000100:00000010:2.0:1773941319.287578:0:5423:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 00000000e6d69441. 00000020:00000010:2.0:1773941319.287582:0:5423:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000cd696590. 00000020:00000010:2.0:1773941319.287586:0:5423:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000008ed3f0ef. 00000020:00000010:2.0:1773941319.287589:0:5423:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000009504a6c. 00000020:00000010:2.0:1773941319.287591:0:5423:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000e72eebc5. 00000040:00000010:2.0:1773941319.287593:0:5423:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000311b26f9. 00000020:00000010:2.0:1773941319.287596:0:5423:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000e3eca39a. 00000020:00000010:2.0:1773941319.287598:0:5423:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 0000000055e91bdd. 00000020:00000010:2.0:1773941319.287601:0:5423:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000593b65fd. 80000000:00000010:2.0:1773941319.287603:0:5423:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000705f4579. 40000000:00000010:2.0:1773941319.287605:0:5423:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000a3328fc6. 00000008:00000010:2.0:1773941319.287608:0:5423:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 0000000027446b89. 00020000:00000010:2.0:1773941319.287611:0:5423:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 0000000090ff605e. 00008000:00000010:2.0:1773941319.287613:0:5423:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 0000000016e01ec5. 00100000:00000010:2.0:1773941319.287616:0:5423:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 0000000069304ec4. 00040000:00000010:2.0:1773941319.287619:0:5423:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000bb78f620. 00040000:00000010:2.0:1773941319.287622:0:5423:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000000bd77e7b. 00040000:00000010:2.0:1773941319.287624:0:5423:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000c32a8374. 00000004:00000010:2.0:1773941319.287626:0:5423:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 0000000052c05d12. 00000004:00000010:2.0:1773941319.287630:0:5423:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 00000000b4bdeec2. 00000004:00000010:2.0:1773941319.287634:0:5423:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 0000000066bfdfea. 00002000:00000010:2.0:1773941319.287637:0:5423:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 0000000008e1ceeb. 00000004:00000010:2.0:1773941319.287639:0:5423:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 000000009dd21e02. 00000080:00000010:2.0:1773941319.287642:0:5423:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000009b60e9d. 00000080:00000010:2.0:1773941319.287644:0:5423:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000378a5d45. 00000100:00000010:2.0:1773941319.287647:0:5423:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 00000000af9e5891. 00000100:00000200:2.0:1773941319.287659:0:5423:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:0.0:1773941319.287675:0:5414:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 00000000522b43ae. 00000100:00000010:3.0:1773941319.287875:0:5424:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 00000000129ee4ce. 00000020:00000010:3.0:1773941319.287880:0:5424:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000003432c377. 00000020:00000010:3.0:1773941319.287885:0:5424:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006ed9f54f. 00000020:00000010:3.0:1773941319.287887:0:5424:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000082baca26. 00000020:00000010:3.0:1773941319.287890:0:5424:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000f473b417. 00000040:00000010:3.0:1773941319.287892:0:5424:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000064d1a610. 00000020:00000010:3.0:1773941319.287895:0:5424:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000e0b6e4b1. 00000020:00000010:3.0:1773941319.287899:0:5424:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000ae1cbe56. 00000020:00000010:3.0:1773941319.287902:0:5424:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000006ef3a7b1. 80000000:00000010:3.0:1773941319.287904:0:5424:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000b416740f. 40000000:00000010:3.0:1773941319.287906:0:5424:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000e3ad5816. 00000008:00000010:3.0:1773941319.287917:0:5424:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 0000000056ab13a7. 00020000:00000010:3.0:1773941319.287920:0:5424:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000001194eb53. 00008000:00000010:3.0:1773941319.287929:0:5424:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 0000000023f6be22. 00100000:00000010:3.0:1773941319.287933:0:5424:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000ad70e582. 00040000:00000010:3.0:1773941319.287936:0:5424:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000fde583a5. 00040000:00000010:3.0:1773941319.287939:0:5424:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000006ac57c94. 00040000:00000010:3.0:1773941319.287947:0:5424:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000b29621c4. 00000004:00000010:3.0:1773941319.287950:0:5424:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000dc2800fe. 00000004:00000010:3.0:1773941319.287953:0:5424:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 0000000087c9b6dd. 00000004:00000010:3.0:1773941319.287955:0:5424:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 0000000080287e39. 00002000:00000010:3.0:1773941319.287957:0:5424:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000bfa808a0. 00000004:00000010:3.0:1773941319.287959:0:5424:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 0000000035a21f39. 00000080:00000010:3.0:1773941319.287968:0:5424:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000568903fd. 00000080:00000010:3.0:1773941319.287978:0:5424:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000cdd17630. 00000100:00000010:3.0:1773941319.287990:0:5424:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 00000000aa3538e5. 00000100:00000200:3.0:1773941319.288007:0:5424:0:(service.c:3086:ptlrpc_main()) service thread 2 (#3) started 00010000:00000010:1.0:1773941319.288026:0:5414:0:(ldlm_lockd.c:3466:ldlm_setup()) kmalloced '(blp)': 168 at 00000000cb48ff97. 00010000:00000010:1.0:1773941319.288164:0:5425:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at 00000000fd05ee1f. 00000020:00000010:1.0:1773941319.288169:0:5425:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c4a51d0c. 00000020:00000010:1.0:1773941319.288173:0:5425:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000a74b96a0. 00000020:00000010:1.0:1773941319.288176:0:5425:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000dbabe6eb. 00000020:00000010:1.0:1773941319.288180:0:5425:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 0000000090b1fc54. 00000020:00000010:1.0:1773941319.288183:0:5425:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000abc42fe6. 80000000:00000010:1.0:1773941319.288185:0:5425:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000cf0f6892. 40000000:00000010:1.0:1773941319.288187:0:5425:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000b9c58dd1. 00100000:00000010:1.0:1773941319.288206:0:5425:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000d48f5941. 00040000:00000010:1.0:1773941319.288215:0:5425:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000279b664e. 00040000:00000010:1.0:1773941319.288217:0:5425:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000052de3db6. 00002000:00000010:1.0:1773941319.288220:0:5425:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 0000000048b927ed. 00000020:00000010:1.0:1773941319.288223:0:5425:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 0000000080295ffd. 00010000:00000010:0.0:1773941319.288365:0:5426:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at 00000000da87164a. 00000020:00000010:0.0:1773941319.288371:0:5426:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000067dc964d. 00000020:00000010:0.0:1773941319.288375:0:5426:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000002f59c161. 00000020:00000010:0.0:1773941319.288378:0:5426:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000003256d688. 00000020:00000010:0.0:1773941319.288381:0:5426:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000eec0ac3d. 00000020:00000010:0.0:1773941319.288383:0:5426:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000007a5a0ce9. 80000000:00000010:0.0:1773941319.288386:0:5426:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000da54ba00. 40000000:00000010:0.0:1773941319.288388:0:5426:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000c80751a8. 00100000:00000010:0.0:1773941319.288399:0:5426:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000000d20cacc. 00040000:00000010:0.0:1773941319.288402:0:5426:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000a87841c1. 00040000:00000010:0.0:1773941319.288405:0:5426:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000006e529e65. 00002000:00000010:0.0:1773941319.288408:0:5426:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000c34f98e1. 00000020:00000010:0.0:1773941319.288410:0:5426:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000b787a22d. 00000020:00000010:1.0:1773941319.288551:0:5427:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000086a2bf56. 00000020:00000010:1.0:1773941319.288556:0:5427:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000117fe22f. 00000040:00000010:1.0:1773941319.288558:0:5427:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000085d9a77a. 00000020:00000010:1.0:1773941319.288561:0:5427:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000de5fe98d. 00000020:00000010:1.0:1773941319.288565:0:5427:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 000000007f9ac0d0. 00000020:00000010:1.0:1773941319.288568:0:5427:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000ed558130. 80000000:00000010:1.0:1773941319.288570:0:5427:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000de1dee26. 40000000:00000010:1.0:1773941319.288572:0:5427:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000f102e456. 00100000:00000010:1.0:1773941319.288575:0:5427:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000f37aa2ae. 00040000:00000010:1.0:1773941319.288577:0:5427:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000000f848b34. 00040000:00000010:1.0:1773941319.288580:0:5427:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000004d1f1f88. 00040000:00000010:1.0:1773941319.288582:0:5427:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000089cbead3. 00000004:00000010:1.0:1773941319.288584:0:5427:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000000f1c6bd1. 00000004:00000010:1.0:1773941319.288588:0:5427:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 00000000e3189086. 00000004:00000010:1.0:1773941319.288590:0:5427:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 0000000039db1efc. 00002000:00000010:1.0:1773941319.288593:0:5427:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000cab8380c. 00000004:00000010:1.0:1773941319.288595:0:5427:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 000000008b3fef43. 00000020:00000010:1.0:1773941319.288597:0:5427:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000001959f04b. 00000001:00000010:0.0:1773941319.288618:0:5414:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 000000008d12975d. 00000001:00000010:0.0:1773941319.288627:0:5414:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000cb9595c5. 00000020:00000010:0.0:1773941319.288645:0:5414:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at 00000000a43b343b. 00000100:00000200:0.0:1773941319.288660:0:5414:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.143@tcp->12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941319.288665:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.288669:0:5414:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.143@tcp primary NID 192.168.204.143@tcp rc 0 00000100:00000010:0.0:1773941319.288673:0:5414:0:(connection.c:113:ptlrpc_connection_get()) kmalloced '(conn)': 40 at 00000000b78916fb. 00010000:00000010:0.0:1773941319.288677:0:5414:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at 000000001d1cc096. 00010000:00000010:0.0:1773941319.288682:0:5414:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at 000000007d40d405. 00000001:00000010:0.0:1773941319.288686:0:5414:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at 00000000fd7e2bda. 00000001:00000010:0.0:1773941319.288690:0:5414:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 16 at 000000001d5ca900. 00000001:00000010:0.0:1773941319.288693:0:5414:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 304 at 00000000cf3db622. 00000001:00000010:0.0:1773941319.288700:0:5414:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 304 at 00000000735fae97. 00000020:00000010:0.0:1773941319.288803:0:5414:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000c21dcdd4. 00000020:00000010:0.0:1773941319.288807:0:5414:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at 00000000f39fc9b1. 00000020:00000010:0.0:1773941319.288877:0:5414:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000a8787ce2. 00000020:00000010:0.0:1773941319.288881:0:5414:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at 000000004069c63a. 00000040:00000010:0.0:1773941319.288897:0:5414:0:(llog_obd.c:26:llog_new_ctxt()) kmalloced '(ctxt)': 168 at 0000000022471819. 00000020:00000010:1.0:1773941319.289178:0:5414:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000e20d0e0f. 00000040:00000010:1.0:1773941319.289184:0:5414:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 000000002c125ee6. 00000020:00000010:1.0:1773941319.289200:0:5414:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 000000009257afff. 00000020:00000010:1.0:1773941319.289206:0:5414:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 0000000020f5eef1. 00040000:00000010:1.0:1773941319.289210:0:5414:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at 000000003d805dca. 00040000:00000010:1.0:1773941319.289214:0:5414:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 00000000555bde7e. 00000020:00000010:1.0:1773941319.289217:0:5414:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000011b24032. 00000020:00000010:1.0:1773941319.289221:0:5414:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000000a4a0442. 00000020:00000010:1.0:1773941319.289224:0:5414:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000097b0e1b6. 00000008:00000010:1.0:1773941319.289227:0:5414:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 000000000ecb35c1. 00020000:00000010:1.0:1773941319.289231:0:5414:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 000000008b33d792. 00008000:00000010:1.0:1773941319.289243:0:5414:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 00000000a3927956. 00000080:00000010:1.0:1773941319.289248:0:5414:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 0000000015ae5784. 00000020:00000010:1.0:1773941319.289252:0:5414:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000001df3009d. 00000020:00000010:1.0:1773941319.289260:0:5414:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 104 at 000000005ab7168d. 00000020:00000010:1.0:1773941319.289274:0:5414:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 000000002e37c6ee. 00000400:00000200:1.0:1773941319.289289:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941319.289293:0:5414:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000010:1.0:1773941319.289314:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000d153cd32. 02000000:00000010:1.0:1773941319.289327:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000024f66857. 00000020:00000010:1.0:1773941319.289362:0:5414:0:(obd_mount.c:561:lustre_start_mgc()) kfreed 'uuid': 40 at 0000000032618b06. 00000020:00000010:0.0:1773941319.289366:0:2351:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000465abf3f. 00000020:00000010:1.0:1773941319.289367:0:5414:0:(obd_mount.c:562:lustre_start_mgc()) kfreed 'data': 192 at 00000000bd9a295e. 00000020:00000010:0.0:1773941319.289372:0:2351:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000eb740600. 00000020:00000010:1.0:1773941319.289373:0:5414:0:(obd_mount.c:563:lustre_start_mgc()) kfreed 'mgcname': 23 at 00000000b34fa120. 80000000:00000010:0.0:1773941319.289376:0:2351:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 0000000088812142. 00000080:00000010:1.0:1773941319.289381:0:5414:0:(llite_lib.c:1399:ll_fill_super()) kmalloced '(cfg)': 88 at 00000000602b1972. 40000000:00000010:0.0:1773941319.289381:0:2351:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 000000008d5292c5. 00000008:00000010:0.0:1773941319.289385:0:2351:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000ebc85ba0. 00000080:00000010:1.0:1773941319.289387:0:5414:0:(llite_lib.c:95:ll_init_sbi()) kmalloced '(sbi)': 1376 at 000000003857970f. 00020000:00000010:0.0:1773941319.289389:0:2351:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 0000000031513bff. 00008000:00000010:0.0:1773941319.289394:0:2351:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000b67d14f7. 00100000:00000010:0.0:1773941319.289398:0:2351:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000004a33288f. 00040000:00000010:0.0:1773941319.289400:0:2351:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000cae76e1b. 00040000:00000010:0.0:1773941319.289402:0:2351:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000ce145078. 00040000:00000010:0.0:1773941319.289405:0:2351:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000a7b8b55c. 00000004:00000010:0.0:1773941319.289409:0:2351:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 0000000044a0584d. 00000004:00000010:0.0:1773941319.289412:0:2351:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 000000000741ee19. 00000004:00000010:0.0:1773941319.289416:0:2351:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000fd4138b5. 00002000:00000010:0.0:1773941319.289419:0:2351:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 000000000c2c34c4. 00000004:00000010:0.0:1773941319.289421:0:2351:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 000000001d1f5c4d. 00000080:00000010:0.0:1773941319.289424:0:2351:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000582aea1e. 00000080:00000010:0.0:1773941319.289427:0:2351:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 0000000054e41761. 00000008:00000010:0.0:1773941319.289430:0:2351:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000acb25cff. 00020000:00000010:0.0:1773941319.289433:0:2351:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 000000009b2ffaf9. 00008000:00000010:0.0:1773941319.289435:0:2351:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 00000000bc81c1fc. 00000080:00000010:0.0:1773941319.289437:0:2351:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 00000000f6b756c2. 02000000:00000010:0.0:1773941319.289449:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000618971b3. 00000400:00000200:0.0:1773941319.289454:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941319.289457:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000080619673. 00000400:00000010:0.0:1773941319.289460:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004bdbaaf4. 00000020:00000010:1.0:1773941319.289463:0:5414:0:(cl_page.c:1380:cl_cache_init()) kmalloced '(cache)': 152 at 00000000a8f89aca. 00000080:00000010:1.0:1773941319.289470:0:5414:0:(llite_lib.c:129:ll_init_sbi()) kmalloced '(sbi->ll_foreign_symlink_prefix)': 6 at 0000000096265173. 00000100:00000200:0.0:1773941319.289471:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241131648, portal 25 00000080:00000010:1.0:1773941319.289477:0:5414:0:(llite_lib.c:136:ll_init_sbi()) kmalloced '(sbi->ll_foreign_symlink_upcall)': 5 at 0000000033476b65. 00000400:00000010:0.0:1773941319.289478:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cc9390c6. 00000100:00000200:0.0:1773941319.289481:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241131648, offset 0 00000400:00000200:0.0:1773941319.289487:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941319.289492:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.289495:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941319.289498:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941319.289502:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941319.289505:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941319.289507:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941319.289509:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [4:4:256:4294967295] 192.168.204.143@tcp LPNI seq info [4:4:8:4294967295] 00000400:00000200:0.0:1773941319.289513:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.289521:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941319.289525:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941319.289528:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000007347e3d6. 00000800:00000200:0.0:1773941319.289534:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941319.289539:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941319.289541:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000007347e3d6 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:0.0:1773941319.289553:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:0.0:1773941319.289557:0:2351:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000009fed569d. 00000100:00000010:0.0:1773941319.289566:0:2351:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000000b8cd59c. 00000800:00000200:3.0:1773941319.289634:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941319.289641:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000007347e3d6. 00000400:00000200:3.0:1773941319.289645:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.289651:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941319.289655:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cc9390c6 00000400:00000010:3.0:1773941319.289658:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000cc9390c6. 00000100:00000200:3.0:1773941319.289665:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000d153cd32 x1860112241131648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941319.290093:0:5414:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000ef48a03c. 00000020:00000010:1.0:1773941319.290098:0:5414:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 1440 at 0000000068776599. 00000020:00000010:1.0:1773941319.290117:0:5414:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000d0adfdb6. 00000020:00000010:1.0:1773941319.290122:0:5414:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 576 at 00000000084216b2. 00000020:00000010:1.0:1773941319.290138:0:5414:0:(obd_mount.c:67:lustre_process_log()) kmalloced '(bufs)': 104 at 000000002f94f82e. 00000020:00000010:1.0:1773941319.290143:0:5414:0:(obd_mount.c:76:lustre_process_log()) kmalloced '(lcfg)': 184 at 00000000fe4ba05e. 00000020:00000010:1.0:1773941319.290147:0:5414:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000555bde7e. 00000020:00000010:1.0:1773941319.290151:0:5414:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000020f5eef1. 00000040:00000010:1.0:1773941319.290154:0:5414:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000002c125ee6. 00000020:00000010:1.0:1773941319.290158:0:5414:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000084fa7f4. 00000020:00000010:1.0:1773941319.290161:0:5414:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 0000000011b24032. 00040000:00000010:1.0:1773941319.290165:0:5414:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000ff580ede. 00040000:00000010:1.0:1773941319.290169:0:5414:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000e20d0e0f. 10000000:00000010:1.0:1773941319.290177:0:5414:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 287 at 00000000f9ec296a. 02000000:00000010:1.0:1773941319.290184:0:5414:0:(sec_config.c:546:sptlrpc_conf_get()) kmalloced '(conf)': 120 at 000000006a4ff6af. 00000100:00000010:1.0:1773941319.290211:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000000582022. 02000000:00000010:1.0:1773941319.290218:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000837136c8. 00010000:00000010:1.0:1773941319.290228:0:5414:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at 00000000e6dcc4a6. 00010000:00000010:1.0:1773941319.290235:0:5414:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at 00000000df586f93. 00000020:00000010:1.0:1773941319.290239:0:5414:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at 00000000aadae163. 00000100:00000010:1.0:1773941319.290253:0:5414:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 000000004a35dc1f. 00000400:00000200:3.0:1773941319.290539:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.290543:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941319.290550:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.290553:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.290557:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.290560:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321600080 00000400:00000200:3.0:1773941319.290564:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.290569:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x20d [1] + 0 00000400:00000200:3.0:1773941319.290581:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.290585:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941319.290591:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000d153cd32 x1860112241131648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941329 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941319.290632:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 0000000080619673. 00000400:00000200:0.0:1773941319.290638:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004bdbaaf4 00000400:00000010:0.0:1773941319.290640:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000004bdbaaf4. 00000100:00000200:0.0:1773941319.290645:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000d153cd32 x1860112241131648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941329 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941319.290655:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000d153cd32 x1860112241131648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941329 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1773941319.290755:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000618971b3. 02000000:00000010:0.0:1773941319.290761:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 0000000024f66857. 00000100:00000010:0.0:1773941319.290772:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000d153cd32. 02000000:00000010:0.0:1773941319.290794:0:5414:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000618971b3. 00000400:00000200:0.0:1773941319.290799:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941319.290802:0:5414:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000080619673. 00000400:00000010:0.0:1773941319.290805:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004bdbaaf4. 00000100:00000200:0.0:1773941319.290808:0:5414:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241131776, portal 25 00000400:00000010:0.0:1773941319.290811:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000e921c2f3. 00000100:00000200:0.0:1773941319.290814:0:5414:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241131776, offset 0 00000400:00000200:0.0:1773941319.290819:0:5414:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941319.290823:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.290828:0:5414:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941319.290831:0:5414:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941319.290833:0:5414:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941319.290836:0:5414:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941319.290838:0:5414:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941319.290842:0:5414:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [5:5:256:4294967295] 192.168.204.143@tcp LPNI seq info [5:5:8:4294967295] 00000400:00000200:0.0:1773941319.290846:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.290854:0:5414:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941319.290858:0:5414:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941319.290861:0:5414:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000098475070. 00000800:00000200:0.0:1773941319.290866:0:5414:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941319.290870:0:5414:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941319.290873:0:5414:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 0000000098475070 type 1, nob 424 niov 1 nkiov 1 00000400:00000200:0.0:1773941319.290886:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941319.290943:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941319.290947:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000098475070. 00000400:00000200:3.0:1773941319.290958:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.290962:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941319.290966:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000e921c2f3 00000400:00000010:3.0:1773941319.290968:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000e921c2f3. 00000100:00000200:3.0:1773941319.290973:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000000582022 x1860112241131776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941319.291730:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.291735:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941319.291741:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.291744:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.291747:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.291751:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 344 into portal 25 MB=0x69bc321600100 00000400:00000200:3.0:1773941319.291754:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.291758:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 344/344 into md 0x215 [1] + 224 00000400:00000200:3.0:1773941319.291766:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.291769:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941319.291775:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000000582022 x1860112241131776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941319.291805:0:5414:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 0000000080619673. 00000400:00000200:0.0:1773941319.291809:0:5414:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004bdbaaf4 00000400:00000010:0.0:1773941319.291811:0:5414:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000004bdbaaf4. 00000100:00000200:0.0:1773941319.291815:0:5414:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000000582022 x1860112241131776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941319.291821:0:5414:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000000582022 x1860112241131776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1773941319.291842:0:5414:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 000000004a35dc1f. 02000000:00000010:0.0:1773941319.291854:0:5414:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000618971b3. 02000000:00000010:0.0:1773941319.291857:0:5414:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000837136c8. 00000100:00000010:0.0:1773941319.291859:0:5414:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000000582022. 10000000:00000010:0.0:1773941319.291865:0:5414:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at 0000000072f83ade. 00000020:00000010:0.0:1773941319.291868:0:5414:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000f678b570. 00000020:00000010:0.0:1773941319.291871:0:5414:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000e27fc341. 00000040:00000010:0.0:1773941319.291875:0:5414:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000031a659b1. 00000020:00000010:0.0:1773941319.291879:0:5414:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000beed777a. 00000020:00000010:0.0:1773941319.291883:0:5414:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000000cda8602. 80000000:00000010:0.0:1773941319.291886:0:5414:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 0000000061dec760. 00040000:00000010:0.0:1773941319.291889:0:5414:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000c28d387a. 20000000:00000010:0.0:1773941319.291891:0:5414:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at 000000004d700348. 00000040:00000010:0.0:1773941319.291896:0:5414:0:(llog.c:42:llog_alloc_handle()) kmalloced '(loghandle)': 352 at 00000000cfcc3898. 00000100:00000010:0.0:1773941319.291908:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000687b9592. 02000000:00000010:0.0:1773941319.291916:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000a43f78e9. 00000100:00000010:0.0:1773941319.291922:0:5414:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000097c40ad1. 02000000:00000010:0.0:1773941319.291929:0:5414:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000007896e949. 00000400:00000200:0.0:1773941319.291932:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941319.291934:0:5414:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000080619673. 00000400:00000010:0.0:1773941319.291937:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004bdbaaf4. 00000100:00000200:0.0:1773941319.291940:0:5414:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241131904, portal 25 00000400:00000010:0.0:1773941319.291944:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000076cbd48d. 00000100:00000200:0.0:1773941319.291945:0:5414:0:(niobuf.c:94:ptl_send_buf()) Sending 512 bytes to portal 26, xid 1860112241131904, offset 0 00000400:00000200:0.0:1773941319.291949:0:5414:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941319.291952:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.291955:0:5414:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941319.291956:0:5414:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941319.291958:0:5414:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941319.291960:0:5414:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941319.291961:0:5414:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941319.291968:0:5414:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [6:6:256:4294967295] 192.168.204.143@tcp LPNI seq info [6:6:8:4294967295] 00000400:00000200:0.0:1773941319.291971:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.291979:0:5414:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941319.291983:0:5414:0:(socklnd_cb.c:1001:ksocknal_send()) sending 512 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941319.291986:0:5414:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000f5f40a4e. 00000800:00000200:0.0:1773941319.291988:0:5414:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941319.291991:0:5414:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941319.291993:0:5414:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000f5f40a4e type 1, nob 608 niov 1 nkiov 1 00000400:00000200:0.0:1773941319.292005:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941319.292057:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941319.292061:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000f5f40a4e. 00000400:00000200:3.0:1773941319.292065:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.292069:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941319.292072:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000076cbd48d 00000400:00000010:3.0:1773941319.292074:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 0000000076cbd48d. 00000100:00000200:3.0:1773941319.292079:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000687b9592 x1860112241131904/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941335 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941319.292658:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.292662:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941319.292666:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.292668:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.292670:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.292677:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 272 into portal 25 MB=0x69bc321600180 00000400:00000200:3.0:1773941319.292680:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.292682:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 272/272 into md 0x21d [1] + 224 00000400:00000200:3.0:1773941319.292689:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.292692:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941319.292696:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000687b9592 x1860112241131904/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941335 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941319.292749:0:5414:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 0000000080619673. 00000400:00000200:1.0:1773941319.292754:0:5414:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004bdbaaf4 00000400:00000010:1.0:1773941319.292756:0:5414:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000004bdbaaf4. 00000100:00000200:1.0:1773941319.292762:0:5414:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000687b9592 x1860112241131904/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941319.292771:0:5414:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000687b9592 x1860112241131904/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941319.292785:0:5414:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 0000000097c40ad1. 02000000:00000010:1.0:1773941319.292790:0:5414:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000007896e949. 02000000:00000010:1.0:1773941319.292792:0:5414:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000a43f78e9. 00000100:00000010:1.0:1773941319.292795:0:5414:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000687b9592. 00000040:00000010:1.0:1773941319.292799:0:5414:0:(llog.c:72:llog_free_handle()) kfreed 'loghandle': 352 at 00000000cfcc3898. 00000020:00000010:1.0:1773941319.292804:0:5414:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000e27fc341. 00000040:00000010:1.0:1773941319.292808:0:5414:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 0000000031a659b1. 00000020:00000010:1.0:1773941319.292810:0:5414:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 00000000beed777a. 00000020:00000010:1.0:1773941319.292814:0:5414:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 000000000cda8602. 80000000:00000010:1.0:1773941319.292817:0:5414:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at 0000000061dec760. 00040000:00000010:1.0:1773941319.292820:0:5414:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 00000000c28d387a. 20000000:00000010:1.0:1773941319.292823:0:5414:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at 000000004d700348. 00000020:00000010:1.0:1773941319.292826:0:5414:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000f678b570. 10000000:00000010:1.0:1773941319.292829:0:5414:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at 0000000072f83ade. 10000000:00000010:1.0:1773941319.292848:0:5414:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 279 at 000000005fde34a5. 10000000:00000010:1.0:1773941319.292853:0:5414:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 286 at 00000000c90e3e94. 10000000:00000010:1.0:1773941319.292858:0:5414:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 285 at 000000000c772e2e. 00000100:00000010:1.0:1773941319.292864:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000bdbed3f. 02000000:00000010:1.0:1773941319.292870:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000837136c8. 00010000:00000010:1.0:1773941319.292885:0:5414:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at 00000000856624cc. 00000100:00000010:1.0:1773941319.292896:0:5414:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000a5e5613c. 02000000:00000010:1.0:1773941319.292905:0:5414:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000045093ebe. 00000400:00000200:1.0:1773941319.292909:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941319.292913:0:5414:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000006a9cf84a. 00000400:00000010:1.0:1773941319.292916:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004bf8f66e. 00000100:00000200:1.0:1773941319.292919:0:5414:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241132032, portal 25 00000400:00000010:1.0:1773941319.292924:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006290fa00. 00000100:00000200:1.0:1773941319.292926:0:5414:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241132032, offset 0 00000400:00000200:1.0:1773941319.292932:0:5414:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941319.292936:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941319.292939:0:5414:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941319.292943:0:5414:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941319.292946:0:5414:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941319.292948:0:5414:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941319.292951:0:5414:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941319.292954:0:5414:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [7:7:256:4294967295] 192.168.204.143@tcp LPNI seq info [7:7:8:4294967295] 00000400:00000200:1.0:1773941319.292958:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941319.292964:0:5414:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941319.292969:0:5414:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941319.292973:0:5414:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000059ad17f4. 00000800:00000200:1.0:1773941319.292976:0:5414:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941319.292979:0:5414:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941319.292981:0:5414:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 0000000059ad17f4 type 1, nob 424 niov 1 nkiov 1 00000400:00000200:1.0:1773941319.292993:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941319.293038:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941319.293042:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000059ad17f4. 00000400:00000200:3.0:1773941319.293045:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.293048:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941319.293051:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006290fa00 00000400:00000010:3.0:1773941319.293052:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000006290fa00. 00000100:00000200:3.0:1773941319.293056:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000000bdbed3f x1860112241132032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941319.293646:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.293650:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941319.293654:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.293656:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.293658:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.293660:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 344 into portal 25 MB=0x69bc321600200 00000400:00000200:3.0:1773941319.293662:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.293664:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 344/344 into md 0x225 [1] + 224 00000400:00000200:3.0:1773941319.293679:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.293683:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941319.293691:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000000bdbed3f x1860112241132032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941319.293758:0:5414:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000006a9cf84a. 00000400:00000200:1.0:1773941319.293764:0:5414:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004bf8f66e 00000400:00000010:1.0:1773941319.293766:0:5414:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000004bf8f66e. 00000100:00000200:1.0:1773941319.293772:0:5414:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000000bdbed3f x1860112241132032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941319.293782:0:5414:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000000bdbed3f x1860112241132032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941319.293807:0:5414:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000a5e5613c. 02000000:00000010:1.0:1773941319.293818:0:5414:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000045093ebe. 02000000:00000010:1.0:1773941319.293822:0:5414:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000837136c8. 00000100:00000010:1.0:1773941319.293825:0:5414:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000000bdbed3f. 10000000:00000010:1.0:1773941319.293832:0:5414:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at 00000000ea630b28. 00000020:00000010:1.0:1773941319.293836:0:5414:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000837136c8. 00000020:00000010:1.0:1773941319.293841:0:5414:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000f678b570. 00000040:00000010:1.0:1773941319.293846:0:5414:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000031a659b1. 00000020:00000010:1.0:1773941319.293850:0:5414:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000174ab7f0. 00000020:00000010:1.0:1773941319.293855:0:5414:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000e27fc341. 80000000:00000010:1.0:1773941319.293869:0:5414:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 000000008b1511c1. 00040000:00000010:1.0:1773941319.293879:0:5414:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000004f4855c. 20000000:00000010:1.0:1773941319.293883:0:5414:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at 0000000007e25331. 00000040:00000010:1.0:1773941319.293889:0:5414:0:(llog.c:42:llog_alloc_handle()) kmalloced '(loghandle)': 352 at 000000009eeedb85. 00000100:00000010:1.0:1773941319.293894:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000bdbed3f. 02000000:00000010:1.0:1773941319.293901:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000b9a8e5d4. 00000100:00000010:1.0:1773941319.293909:0:5414:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000a5e5613c. 02000000:00000010:1.0:1773941319.293918:0:5414:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 0000000034b43cef. 00000400:00000200:1.0:1773941319.293923:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941319.293926:0:5414:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000006a9cf84a. 00000400:00000010:1.0:1773941319.293931:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004bf8f66e. 00000100:00000200:1.0:1773941319.293934:0:5414:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241132160, portal 25 00000400:00000010:1.0:1773941319.293939:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000270dd059. 00000100:00000200:1.0:1773941319.293941:0:5414:0:(niobuf.c:94:ptl_send_buf()) Sending 512 bytes to portal 26, xid 1860112241132160, offset 0 00000400:00000200:1.0:1773941319.293948:0:5414:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941319.293953:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941319.293957:0:5414:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941319.293960:0:5414:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941319.293963:0:5414:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941319.293966:0:5414:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941319.293968:0:5414:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941319.293978:0:5414:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [8:8:256:4294967295] 192.168.204.143@tcp LPNI seq info [8:8:8:4294967295] 00000400:00000200:1.0:1773941319.293982:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941319.293990:0:5414:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941319.293994:0:5414:0:(socklnd_cb.c:1001:ksocknal_send()) sending 512 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941319.293998:0:5414:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000076643c64. 00000800:00000200:1.0:1773941319.294002:0:5414:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941319.294006:0:5414:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941319.294009:0:5414:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 0000000076643c64 type 1, nob 608 niov 1 nkiov 1 00000400:00000200:1.0:1773941319.294025:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941319.294604:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941319.294608:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000076643c64. 00000400:00000200:3.0:1773941319.294612:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.294621:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941319.294625:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000270dd059 00000400:00000010:3.0:1773941319.294627:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000270dd059. 00000100:00000200:3.0:1773941319.294632:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000000bdbed3f x1860112241132160/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941335 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941319.295252:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.295257:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941319.295262:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.295265:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.295269:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.295271:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 272 into portal 25 MB=0x69bc321600280 00000400:00000200:3.0:1773941319.295276:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.295280:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 272/272 into md 0x22d [1] + 224 00000400:00000200:3.0:1773941319.295302:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.295306:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941319.295311:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000000bdbed3f x1860112241132160/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941335 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941319.295355:0:5414:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000006a9cf84a. 00000400:00000200:0.0:1773941319.295361:0:5414:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004bf8f66e 00000400:00000010:0.0:1773941319.295363:0:5414:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000004bf8f66e. 00000100:00000200:0.0:1773941319.295370:0:5414:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000000bdbed3f x1860112241132160/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941319.295380:0:5414:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000000bdbed3f x1860112241132160/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1773941319.295397:0:5414:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000a5e5613c. 02000000:00000010:0.0:1773941319.295405:0:5414:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 0000000034b43cef. 02000000:00000010:0.0:1773941319.295408:0:5414:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000b9a8e5d4. 00000100:00000010:0.0:1773941319.295411:0:5414:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000000bdbed3f. 00000040:00000010:0.0:1773941319.295418:0:5414:0:(llog.c:412:llog_init_handle()) kmalloced '(llh)': 8192 at 00000000919865ef. 00000100:00000010:0.0:1773941319.295423:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005151efc1. 02000000:00000010:0.0:1773941319.295431:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000055fda90. 00000100:00000010:0.0:1773941319.295436:0:5414:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 000000007e36aa11. 02000000:00000010:0.0:1773941319.295455:0:5414:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at 000000008d09144f. 00000400:00000200:0.0:1773941319.295460:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941319.295464:0:5414:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c405db14. 00000400:00000010:0.0:1773941319.295467:0:5414:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at 00000000bd183e77. 00000100:00000200:0.0:1773941319.295471:0:5414:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112241132288, portal 25 00000400:00000010:0.0:1773941319.295474:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000040bfedff. 00000100:00000200:0.0:1773941319.295483:0:5414:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112241132288, offset 0 00000400:00000200:0.0:1773941319.295489:0:5414:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941319.295493:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.295498:0:5414:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941319.295501:0:5414:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941319.295503:0:5414:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941319.295506:0:5414:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941319.295508:0:5414:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941319.295511:0:5414:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [9:9:256:4294967295] 192.168.204.143@tcp LPNI seq info [9:9:8:4294967295] 00000400:00000200:0.0:1773941319.295521:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.295528:0:5414:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941319.295533:0:5414:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941319.295537:0:5414:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000033a5ccb4. 00000800:00000200:0.0:1773941319.295543:0:5414:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941319.295548:0:5414:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941319.295551:0:5414:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 0000000033a5ccb4 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:0.0:1773941319.295565:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941319.295622:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941319.295626:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000033a5ccb4. 00000400:00000200:3.0:1773941319.295631:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.295635:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941319.295639:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000040bfedff 00000400:00000010:3.0:1773941319.295641:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 0000000040bfedff. 00000100:00000200:3.0:1773941319.295647:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005151efc1 x1860112241132288/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941335 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941319.296389:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.296393:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941319.296398:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.296400:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.296402:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.296403:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 8416 into portal 25 MB=0x69bc321600300 00000400:00000200:3.0:1773941319.296411:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.296456:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 8416/8416 into md 0x235 [4] + 224 00000400:00000200:3.0:1773941319.296466:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.296468:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941319.296473:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005151efc1 x1860112241132288/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941335 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941319.296515:0:5414:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c405db14. 00000400:00000200:0.0:1773941319.296521:0:5414:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000bd183e77 00000400:00000010:0.0:1773941319.296523:0:5414:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at 00000000bd183e77. 00000100:00000200:0.0:1773941319.296529:0:5414:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005151efc1 x1860112241132288/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941319.296538:0:5414:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005151efc1 x1860112241132288/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1773941319.296559:0:5414:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 000000007e36aa11. 02000000:00000010:0.0:1773941319.296566:0:5414:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at 000000008d09144f. 02000000:00000010:0.0:1773941319.296575:0:5414:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000055fda90. 00000100:00000010:0.0:1773941319.296578:0:5414:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005151efc1. 00000040:00000010:0.0:1773941319.296585:0:5414:0:(llog.c:945:llog_process_or_fork()) kmalloced '(lpi)': 96 at 00000000a35a07c3. 00000020:00000010:2.0:1773941319.296766:0:5433:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000355b1b86. 00000020:00000010:2.0:1773941319.296772:0:5433:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000cc6806d0. 00000040:00000010:2.0:1773941319.296775:0:5433:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000ceb8ec80. 00000020:00000010:2.0:1773941319.296778:0:5433:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000003a503b55. 00000020:00000010:2.0:1773941319.296781:0:5433:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 000000007dc14f5e. 00000020:00000010:2.0:1773941319.296786:0:5433:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 0000000056cad018. 80000000:00000010:2.0:1773941319.296793:0:5433:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 000000001a3861b4. 40000000:00000010:2.0:1773941319.296795:0:5433:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 000000003e930e78. 00100000:00000010:2.0:1773941319.296799:0:5433:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000a96b4ca5. 00040000:00000010:2.0:1773941319.296802:0:5433:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000009aea171e. 00040000:00000010:2.0:1773941319.296805:0:5433:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000003074a38a. 00040000:00000010:2.0:1773941319.296808:0:5433:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000c56bbe08. 00000004:00000010:2.0:1773941319.296810:0:5433:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000006d5cee44. 00000004:00000010:2.0:1773941319.296815:0:5433:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 000000001827d6c7. 00000004:00000010:2.0:1773941319.296817:0:5433:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 000000006b6b69dc. 00002000:00000010:2.0:1773941319.296820:0:5433:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 0000000073644ac5. 00000004:00000010:2.0:1773941319.296823:0:5433:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 000000000fe7dd4f. 00000020:00000010:2.0:1773941319.296827:0:5433:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000b28c71cf. 00000040:00000010:2.0:1773941319.296833:0:5433:0:(llog.c:570:llog_process_thread()) kmalloced '(buf)': 8192 at 0000000007f45625. 00000100:00000010:2.0:1773941319.296843:0:5433:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000ae998bae. 02000000:00000010:2.0:1773941319.296848:0:5433:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000f7a554a1. 00000100:00000010:2.0:1773941319.296853:0:5433:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000fef06286. 02000000:00000010:2.0:1773941319.296864:0:5433:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at 0000000019cb83db. 00000400:00000200:2.0:1773941319.296868:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941319.296870:0:5433:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000db9eafd5. 00000400:00000010:2.0:1773941319.296873:0:5433:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at 00000000b27eb54c. 00000100:00000200:2.0:1773941319.296876:0:5433:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112241132416, portal 25 00000400:00000010:2.0:1773941319.296887:0:5433:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000007af3d3cd. 00000100:00000200:2.0:1773941319.296890:0:5433:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112241132416, offset 0 00000400:00000200:2.0:1773941319.296897:0:5433:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941319.296902:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.296906:0:5433:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941319.296909:0:5433:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941319.296912:0:5433:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941319.296915:0:5433:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941319.296917:0:5433:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941319.296919:0:5433:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [10:10:256:4294967295] 192.168.204.143@tcp LPNI seq info [10:10:8:4294967295] 00000400:00000200:2.0:1773941319.296923:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.296929:0:5433:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941319.296942:0:5433:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941319.296945:0:5433:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000baad2b1f. 00000800:00000200:2.0:1773941319.296950:0:5433:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941319.296954:0:5433:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941319.296957:0:5433:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000baad2b1f type 1, nob 368 niov 1 nkiov 1 00000400:00000200:2.0:1773941319.296967:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941319.297015:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941319.297018:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000baad2b1f. 00000400:00000200:3.0:1773941319.297021:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.297024:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941319.297026:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000007af3d3cd 00000400:00000010:3.0:1773941319.297027:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000007af3d3cd. 00000100:00000200:3.0:1773941319.297031:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000ae998bae x1860112241132416/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941319.297697:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.297700:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941319.297718:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.297720:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.297724:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.297726:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 8472 into portal 25 MB=0x69bc321600380 00000400:00000200:3.0:1773941319.297772:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.297775:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 8472/8472 into md 0x23d [4] + 224 00000400:00000200:2.0:1773941319.297836:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941319.297841:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941319.297845:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000ae998bae x1860112241132416/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941319.297884:0:5433:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000db9eafd5. 00000400:00000200:3.0:1773941319.297889:0:5433:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b27eb54c 00000400:00000010:3.0:1773941319.297890:0:5433:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at 00000000b27eb54c. 00000100:00000200:3.0:1773941319.297896:0:5433:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000ae998bae x1860112241132416/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941319.297904:0:5433:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000ae998bae x1860112241132416/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941319.297919:0:5433:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000fef06286. 02000000:00000010:3.0:1773941319.297932:0:5433:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at 0000000019cb83db. 02000000:00000010:3.0:1773941319.297938:0:5433:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000f7a554a1. 00000100:00000010:3.0:1773941319.297941:0:5433:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000ae998bae. 00000020:00000010:3.0:1773941319.297958:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000fd41e7e9. 00000020:00000010:3.0:1773941319.297961:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000fd41e7e9. 00000020:00000010:3.0:1773941319.297965:0:5433:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 0000000086edde8e. 00000020:00000010:3.0:1773941319.297969:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 128 at 000000008e0b0091. 00000020:00000010:3.0:1773941319.297982:0:5433:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at 000000001478057f. 00000020:00000010:3.0:1773941319.297988:0:5433:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 00000000bcba8085. 00000020:00000010:3.0:1773941319.297999:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 128 at 000000008e0b0091. 00000020:00000010:3.0:1773941319.298001:0:5433:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at 0000000086edde8e. 00000020:00000010:3.0:1773941319.298004:0:5433:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 0000000086edde8e. 00000020:00000010:3.0:1773941319.298006:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 160 at 0000000007e527fd. 00000001:00000010:3.0:1773941319.298024:0:5433:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 00000000fd41e7e9. 00000001:00000010:3.0:1773941319.298027:0:5433:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 000000003c5af960. 00000001:00000010:3.0:1773941319.298029:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000ad8ecbb4. 00000001:00000010:3.0:1773941319.298031:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000df4f4250. 00000001:00000010:3.0:1773941319.298033:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000008266b74c. 00000001:00000010:3.0:1773941319.298035:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000001adab740. 00000020:00000010:3.0:1773941319.298037:0:5433:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000024e06ef9. 00000020:00000010:3.0:1773941319.298040:0:5433:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000008e0b0091. 00000020:00000010:3.0:1773941319.298048:0:5433:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000000bf6bd2. 00000008:00000010:3.0:1773941319.298059:0:5433:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000c8661c5e. 00020000:00000010:3.0:1773941319.298070:0:5433:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 00000000948107ec. 00008000:00000010:3.0:1773941319.298082:0:5433:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 0000000088a31a61. 00000080:00000010:3.0:1773941319.298093:0:5433:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 000000007dc1df9a. 00000020:00000010:3.0:1773941319.298096:0:5433:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000aac61184. 00000020:00000010:3.0:1773941319.298098:0:5433:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000000409cda. 00000020:00000010:3.0:1773941319.298101:0:5433:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000a635135a. 00000008:00000010:3.0:1773941319.298105:0:5433:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 0000000074ea5404. 00020000:00000010:3.0:1773941319.298107:0:5433:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000002149e0f1. 00008000:00000010:3.0:1773941319.298110:0:5433:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000b3580e99. 00000080:00000010:3.0:1773941319.298116:0:5433:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000001e799344. 00000080:00000010:3.0:1773941319.298118:0:5433:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000002abe1461. 00020000:00000010:3.0:1773941319.298122:0:5433:0:(lov_dev.c:599:lov_device_alloc()) kmalloced '(ld)': 632 at 00000000215e33a4. 00000020:00000010:3.0:1773941319.298137:0:5433:0:(lu_tgt_pool.c:52:lu_tgt_pool_init()) kmalloced '(op->op_array)': 8 at 00000000574ceb9a. 00020000:00000010:3.0:1773941319.298260:0:5433:0:(lov_dev.c:616:lov_device_alloc()) kmalloced '(ld->ld_md_tgts)': 4096 at 0000000009c6b459. 00020000:00000010:3.0:1773941319.298265:0:5433:0:(lov_dev.c:623:lov_device_alloc()) kmalloced '(ld->ld_lov->lov_mdc_tgts)': 4096 at 00000000da63cfcf. 00000020:00000010:3.0:1773941319.298291:0:5433:0:(lu_object.c:1154:lu_site_init()) kmalloced '(s->ls_bkts)': 18432 at 000000004229df9c. 00000020:00000010:3.0:1773941319.298300:0:5433:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000139ed1d9. 00000020:00000010:3.0:1773941319.298302:0:5433:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 192 at 000000000fde465a. 00000020:00000010:3.0:1773941319.298308:0:5433:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 0000000047525c39. 00000040:00000010:3.0:1773941319.298310:0:5433:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000a051c1af. 00000020:00000010:3.0:1773941319.298312:0:5433:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000004082f861. 00000020:00000010:3.0:1773941319.298315:0:5433:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000744d0536. 00000020:00000010:3.0:1773941319.298318:0:5433:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 0000000025627089. 80000000:00000010:3.0:1773941319.298320:0:5433:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 000000004365fcfe. 40000000:00000010:3.0:1773941319.298322:0:5433:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000ce596634. 00000008:00000010:3.0:1773941319.298324:0:5433:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000853f6d10. 00020000:00000010:3.0:1773941319.298326:0:5433:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000b0e1ec58. 00008000:00000010:3.0:1773941319.298330:0:5433:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000007cf4b157. 00100000:00000010:3.0:1773941319.298334:0:5433:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 0000000092bc8296. 00040000:00000010:3.0:1773941319.298337:0:5433:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 0000000086faabc1. 00040000:00000010:3.0:1773941319.298340:0:5433:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000bd37d6af. 00040000:00000010:3.0:1773941319.298343:0:5433:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000008f801ab8. 00000004:00000010:3.0:1773941319.298346:0:5433:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000677b6358. 00000004:00000010:3.0:1773941319.298350:0:5433:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 000000000457d956. 00000004:00000010:3.0:1773941319.298353:0:5433:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 000000009d258142. 00002000:00000010:3.0:1773941319.298357:0:5433:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 000000006925849c. 00000004:00000010:3.0:1773941319.298360:0:5433:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 000000008c3f3b1b. 00000080:00000010:3.0:1773941319.298363:0:5433:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000003d0f76ea. 00000080:00000010:3.0:1773941319.298366:0:5433:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000009f2e4bb4. 00000020:00000010:3.0:1773941319.298370:0:5433:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000aa5dc4a7. 00000040:00000010:3.0:1773941319.298373:0:5433:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000006d40ba3d. 00000020:00000010:3.0:1773941319.298385:0:5433:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000c01f9c24. 00000020:00000010:3.0:1773941319.298388:0:5433:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 000000005a6b71a7. 00000020:00000010:3.0:1773941319.298390:0:5433:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000ad82944b. 80000000:00000010:3.0:1773941319.298392:0:5433:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000feef275d. 40000000:00000010:3.0:1773941319.298393:0:5433:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000cd44e6f6. 00000008:00000010:3.0:1773941319.298396:0:5433:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000dd0f8d9b. 00020000:00000010:3.0:1773941319.298399:0:5433:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000971a131b. 00008000:00000010:3.0:1773941319.298402:0:5433:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000009a81205c. 00100000:00000010:3.0:1773941319.298404:0:5433:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000001d6df1b9. 00040000:00000010:3.0:1773941319.298406:0:5433:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000006bd9e76a. 00040000:00000010:3.0:1773941319.298408:0:5433:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000eec12e16. 00040000:00000010:3.0:1773941319.298410:0:5433:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000d38db563. 00000004:00000010:3.0:1773941319.298413:0:5433:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 0000000075484ed1. 00000004:00000010:3.0:1773941319.298416:0:5433:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 0000000090273caa. 00000004:00000010:3.0:1773941319.298419:0:5433:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 000000001fea1afd. 00002000:00000010:3.0:1773941319.298422:0:5433:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 0000000087ef9f05. 00000004:00000010:3.0:1773941319.298426:0:5433:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 0000000083cd1eaf. 00000080:00000010:3.0:1773941319.298429:0:5433:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000054f2eb41. 00000080:00000010:3.0:1773941319.298432:0:5433:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000002db99900. 00000020:00000010:3.0:1773941319.298436:0:5433:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 0000000000409cda. 00000020:00000010:3.0:1773941319.298441:0:5433:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000a635135a. 00000008:00000010:3.0:1773941319.298445:0:5433:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 0000000074ea5404. 00020000:00000010:3.0:1773941319.298449:0:5433:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000002149e0f1. 00008000:00000010:3.0:1773941319.298452:0:5433:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000b3580e99. 00000080:00000010:3.0:1773941319.298455:0:5433:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 000000001e799344. 00000080:00000010:3.0:1773941319.298459:0:5433:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 000000002abe1461. 00000020:00000010:3.0:1773941319.298462:0:5433:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000aac61184. 00000020:00000010:3.0:1773941319.298466:0:5433:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000008e0b0091. 00000020:00000010:3.0:1773941319.298468:0:5433:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000000bf6bd2. 00000008:00000010:3.0:1773941319.298471:0:5433:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000c8661c5e. 00020000:00000010:3.0:1773941319.298475:0:5433:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 00000000948107ec. 00008000:00000010:3.0:1773941319.298477:0:5433:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 0000000088a31a61. 00000080:00000010:3.0:1773941319.298479:0:5433:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 000000007dc1df9a. 00000020:00000010:3.0:1773941319.298493:0:5433:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000024e06ef9. 00000020:00000010:3.0:1773941319.298495:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 160 at 0000000007e527fd. 00000020:00000010:3.0:1773941319.298498:0:5433:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at 0000000086edde8e. 00000020:00000010:3.0:1773941319.298502:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 0000000002b7c9d7. 00000020:00000010:3.0:1773941319.298503:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 0000000002b7c9d7. 00000020:00000010:3.0:1773941319.298506:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 0000000002b7c9d7. 00000020:00000010:3.0:1773941319.298507:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 0000000002b7c9d7. 00000020:00000010:3.0:1773941319.298508:0:5433:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 0000000086edde8e. 00000020:00000010:3.0:1773941319.298510:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 128 at 000000008e0b0091. 00000020:00000010:3.0:1773941319.298514:0:5433:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at 000000002b1c97a4. 00000020:00000010:3.0:1773941319.298518:0:5433:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 000000003c1fc88a. 00000020:00000010:3.0:1773941319.298523:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 128 at 000000008e0b0091. 00000020:00000010:3.0:1773941319.298524:0:5433:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at 0000000086edde8e. 00000020:00000010:3.0:1773941319.298527:0:5433:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 0000000086edde8e. 00000020:00000010:3.0:1773941319.298529:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 160 at 0000000007e527fd. 00000001:00000010:3.0:1773941319.298550:0:5433:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 0000000002b7c9d7. 00000001:00000010:3.0:1773941319.298552:0:5433:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 000000007aa7e94e. 00000001:00000010:3.0:1773941319.298554:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 0000000024e06ef9. 00000001:00000010:3.0:1773941319.298555:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000aac61184. 00000001:00000010:3.0:1773941319.298556:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 0000000000409cda. 00000001:00000010:3.0:1773941319.298558:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000c38e91ca. 00000020:00000010:3.0:1773941319.298560:0:5433:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ef288c1d. 00000020:00000010:3.0:1773941319.298562:0:5433:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000008e0b0091. 00000020:00000010:3.0:1773941319.298565:0:5433:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000000bf6bd2. 00000008:00000010:3.0:1773941319.298567:0:5433:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000c8661c5e. 00020000:00000010:3.0:1773941319.298569:0:5433:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 00000000948107ec. 00008000:00000010:3.0:1773941319.298572:0:5433:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 0000000088a31a61. 00000080:00000010:3.0:1773941319.298574:0:5433:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 000000007dc1df9a. 00000020:00000010:3.0:1773941319.298578:0:5433:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b7fbafb1. 00000020:00000010:3.0:1773941319.298580:0:5433:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000080c07d32. 00000040:00000010:3.0:1773941319.298582:0:5433:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000f345fff9. 00000020:00000010:3.0:1773941319.298590:0:5433:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000519e3365. 00000020:00000010:3.0:1773941319.298593:0:5433:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 0000000012c246e7. 00040000:00000010:3.0:1773941319.298596:0:5433:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000edfdd59b. 00040000:00000010:3.0:1773941319.298599:0:5433:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000c6d68ffc. 00800000:00000010:3.0:1773941319.298602:0:5433:0:(lmv_obd.c:1219:lmv_device_alloc()) kmalloced '(lu)': 64 at 00000000b2457b4b. 00000020:00000010:3.0:1773941319.298693:0:5433:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000e15d456d. 00000020:00000010:3.0:1773941319.298696:0:5433:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 512 at 000000007a60632d. 80000000:00000010:3.0:1773941319.298735:0:5433:0:(fld_cache.c:47:fld_cache_init()) kmalloced '(cache)': 168 at 0000000084957012. 00800000:00000010:3.0:1773941319.298764:0:5433:0:(lmv_obd.c:1279:lmv_device_alloc()) kmalloced '(pat)': 280 at 00000000e9cf69ef. 00800000:00000010:3.0:1773941319.298767:0:5433:0:(lmv_obd.c:1288:lmv_device_alloc()) kmalloced '(pat)': 280 at 00000000c9461946. 00000020:00000010:3.0:1773941319.298769:0:5433:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 0000000080c07d32. 00000040:00000010:3.0:1773941319.298772:0:5433:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 00000000f345fff9. 00000020:00000010:3.0:1773941319.298773:0:5433:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 00000000519e3365. 00000020:00000010:3.0:1773941319.298776:0:5433:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 0000000012c246e7. 00040000:00000010:3.0:1773941319.298778:0:5433:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at 00000000edfdd59b. 00040000:00000010:3.0:1773941319.298780:0:5433:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 00000000c6d68ffc. 00000020:00000010:3.0:1773941319.298783:0:5433:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000b7fbafb1. 00000020:00000010:3.0:1773941319.298785:0:5433:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000008e0b0091. 00000020:00000010:3.0:1773941319.298787:0:5433:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000000bf6bd2. 00000008:00000010:3.0:1773941319.298789:0:5433:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000c8661c5e. 00020000:00000010:3.0:1773941319.298791:0:5433:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 00000000948107ec. 00008000:00000010:3.0:1773941319.298792:0:5433:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 0000000088a31a61. 00000080:00000010:3.0:1773941319.298794:0:5433:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 000000007dc1df9a. 00000020:00000010:3.0:1773941319.298795:0:5433:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000ef288c1d. 00000020:00000010:3.0:1773941319.298797:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 160 at 0000000007e527fd. 00000020:00000010:3.0:1773941319.298799:0:5433:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at 0000000086edde8e. 00000020:00000010:3.0:1773941319.298804:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000edfdd59b. 00000020:00000010:3.0:1773941319.298805:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000edfdd59b. 00000020:00000010:3.0:1773941319.298809:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000edfdd59b. 00000020:00000010:3.0:1773941319.298810:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000edfdd59b. 00000020:00000010:3.0:1773941319.298814:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 96 at 00000000a635135a. 00000020:00000010:3.0:1773941319.298817:0:5433:0:(obd_config.c:1186:class_add_profile()) kmalloced '(lprof)': 48 at 0000000086edde8e. 00000020:00000010:3.0:1773941319.298824:0:5433:0:(obd_config.c:1192:class_add_profile()) kmalloced '(lprof->lp_profile)': 14 at 0000000074bfdeab. 00000020:00000010:3.0:1773941319.298827:0:5433:0:(obd_config.c:1198:class_add_profile()) kmalloced '(lprof->lp_dt)': 14 at 000000004631aad9. 00000020:00000010:3.0:1773941319.298830:0:5433:0:(obd_config.c:1205:class_add_profile()) kmalloced '(lprof->lp_md)': 14 at 000000000102a9c4. 00000020:00000010:3.0:1773941319.298831:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 96 at 00000000a635135a. 00000020:00000010:3.0:1773941319.298835:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000edfdd59b. 00000020:00000010:3.0:1773941319.298837:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000edfdd59b. 00000020:00000010:3.0:1773941319.298841:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000edfdd59b. 00000020:00000010:3.0:1773941319.298842:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000edfdd59b. 00000020:00000010:3.0:1773941319.298843:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at 00000000ea415e53. 00000100:00000010:3.0:1773941319.298845:0:5433:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at 00000000ed7ceb42. 00000400:00000200:3.0:1773941319.298848:0:5433:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1773941319.298852:0:5433:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.143@tcp 00000400:00000200:3.0:1773941319.298855:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.298857:0:5433:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.143@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1773941319.298860:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.298864:0:5433:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1773941319.298867:0:5433:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp: 0. pending discovery 00000400:00000200:3.0:1773941319.298869:0:5433:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.143@tcp 00000100:00000010:3.0:1773941319.298870:0:5433:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at 00000000ed7ceb42. 00000020:00000010:3.0:1773941319.298872:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 64 at 00000000ea415e53. 00000020:00000010:3.0:1773941319.298875:0:5433:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 00000000ea415e53. 00000020:00000010:3.0:1773941319.298878:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 0000000007e527fd. 00000020:00000010:3.0:1773941319.298883:0:5433:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at 000000003d0dd29c. 00000020:00000010:3.0:1773941319.298887:0:5433:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 00000000519e3365. 00000020:00000010:3.0:1773941319.298894:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 0000000007e527fd. 00000020:00000010:3.0:1773941319.298896:0:5433:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at 00000000ea415e53. 00000020:00000010:3.0:1773941319.298899:0:5433:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 00000000ea415e53. 00000020:00000010:3.0:1773941319.298902:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 0000000007e527fd. 00000001:00000010:3.0:1773941319.298914:0:5433:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 00000000edfdd59b. 00000001:00000010:3.0:1773941319.298917:0:5433:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 00000000db112128. 00000001:00000010:3.0:1773941319.298919:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 0000000068e88519. 00000001:00000010:3.0:1773941319.298922:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 0000000074a75b9b. 00000001:00000010:3.0:1773941319.298929:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000afae76cf. 00000001:00000010:3.0:1773941319.298931:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000ef288c1d. 00000020:00000010:3.0:1773941319.298933:0:5433:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b7fbafb1. 00000020:00000010:3.0:1773941319.298936:0:5433:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000008e0b0091. 00000020:00000010:3.0:1773941319.298938:0:5433:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000000bf6bd2. 00000008:00000010:3.0:1773941319.298940:0:5433:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000c8661c5e. 00020000:00000010:3.0:1773941319.298942:0:5433:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 00000000948107ec. 00008000:00000010:3.0:1773941319.298945:0:5433:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 0000000088a31a61. 00000080:00000010:3.0:1773941319.298948:0:5433:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 000000007dc1df9a. 00000020:00000010:3.0:1773941319.298951:0:5433:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c6d68ffc. 00000020:00000010:3.0:1773941319.298953:0:5433:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000012c246e7. 00000020:00000010:3.0:1773941319.298954:0:5433:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000a635135a. 00000008:00000010:3.0:1773941319.298956:0:5433:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 0000000074ea5404. 00020000:00000010:3.0:1773941319.298958:0:5433:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000002149e0f1. 00008000:00000010:3.0:1773941319.298960:0:5433:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000b3580e99. 00000080:00000010:3.0:1773941319.298963:0:5433:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000001e799344. 00000080:00000010:3.0:1773941319.298965:0:5433:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000002abe1461. 00000002:00000010:3.0:1773941319.298968:0:5433:0:(mdc_dev.c:1726:mdc_device_alloc()) kmalloced '(osc)': 104 at 00000000e074b916. 00010000:00000010:3.0:1773941319.298977:0:5433:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at 000000002b923abc. 00010000:00000010:3.0:1773941319.298982:0:5433:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at 0000000097711bcd. 00010000:00000010:3.0:1773941319.298985:0:5433:0:(ldlm_lib.c:522:client_obd_setup()) kmalloced '(cli->cl_mod_tag_bitmap)': 64 at 000000009583afde. 00000020:00000010:3.0:1773941319.298988:0:5433:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at 0000000063f3d887. 00000100:00000200:3.0:1773941319.298997:0:5433:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.143@tcp->12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941319.299000:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.299003:0:5433:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.143@tcp primary NID 192.168.204.143@tcp rc 0 00010000:00000010:3.0:1773941319.299007:0:5433:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at 0000000074c3a1c8. 00010000:00000010:3.0:1773941319.299011:0:5433:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at 00000000450ae5c8. 00000001:00000010:3.0:1773941319.299014:0:5433:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at 00000000f345fff9. 00000001:00000010:3.0:1773941319.299017:0:5433:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 256 at 00000000c0286ada. 00000001:00000010:3.0:1773941319.299054:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000fab2fc97. 00000001:00000010:3.0:1773941319.299091:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000046829050. 00000001:00000010:3.0:1773941319.299134:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000026e120df. 00000001:00000010:3.0:1773941319.299198:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000003ab2e79a. 00000001:00000010:3.0:1773941319.299235:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000c24094e2. 00000001:00000010:3.0:1773941319.299273:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000a2879a42. 00000001:00000010:3.0:1773941319.299317:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000b48295b9. 00000001:00000010:3.0:1773941319.299369:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000fbdc026a. 00000001:00000010:3.0:1773941319.299411:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000335995c2. 00000001:00000010:3.0:1773941319.299468:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000137376cd. 00000001:00000010:3.0:1773941319.299516:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000a3f8745e. 00000001:00000010:3.0:1773941319.299558:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000073fcf243. 00000001:00000010:3.0:1773941319.299596:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000eba4d898. 00000001:00000010:3.0:1773941319.299637:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000dde97817. 00000001:00000010:3.0:1773941319.299680:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000182e0d04. 00000001:00000010:3.0:1773941319.299740:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000cec037df. 00000001:00000010:3.0:1773941319.299788:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000018d743c7. 00000001:00000010:3.0:1773941319.299830:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000000fe73d3e. 00000001:00000010:3.0:1773941319.299869:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000058de2a2d. 00000001:00000010:3.0:1773941319.299908:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000297c2554. 00000001:00000010:3.0:1773941319.299948:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000003c13d3c5. 00000001:00000010:3.0:1773941319.299987:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000b20d3a9d. 00000001:00000010:3.0:1773941319.300024:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000077b39f0f. 00000001:00000010:3.0:1773941319.300066:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000003e0672e3. 00000001:00000010:3.0:1773941319.300096:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000009a63a4bf. 00000001:00000010:3.0:1773941319.300129:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000db0ebf17. 00000001:00000010:3.0:1773941319.300171:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000009cc68d1a. 00000001:00000010:3.0:1773941319.300218:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000000c217b03. 00000001:00000010:3.0:1773941319.300251:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000005802895a. 00000001:00000010:3.0:1773941319.300286:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000036524af2. 00000001:00000010:3.0:1773941319.300331:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000dafc5f27. 00000001:00000010:3.0:1773941319.300372:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000fdafdb85. 00000020:00000010:3.0:1773941319.300423:0:5433:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 000000007a18c44d. 00000020:00000010:3.0:1773941319.300426:0:5433:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at 000000009bdf5fe8. 00000020:00000010:3.0:1773941319.300490:0:5433:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 000000005c179efe. 00000020:00000010:3.0:1773941319.300492:0:5433:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at 0000000080c07d32. 00000020:00000010:3.0:1773941319.300624:0:5433:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 000000003ddf9598. 00000020:00000010:3.0:1773941319.300627:0:5433:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 512 at 00000000be7cd082. 00000020:00000010:3.0:1773941319.300648:0:5433:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000bb12464a. 00000020:00000010:3.0:1773941319.300651:0:5433:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 00000000aeff85ee. 00000040:00000010:3.0:1773941319.300666:0:5433:0:(llog_obd.c:26:llog_new_ctxt()) kmalloced '(ctxt)': 168 at 000000005b88affe. 00000002:00000010:3.0:1773941319.300673:0:5433:0:(mdc_changelog.c:816:mdc_changelog_cdev_init()) kmalloced '(entry)': 1304 at 000000006b40d708. 00000100:00000010:3.0:1773941319.300740:0:33:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000000b8cd59c. 00000100:00000010:2.0:1773941319.301115:0:758:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000009fed569d. 00000020:00000010:3.0:1773941319.301210:0:5433:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 0000000012c246e7. 00000020:00000010:3.0:1773941319.301214:0:5433:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000a635135a. 00000008:00000010:3.0:1773941319.301217:0:5433:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 0000000074ea5404. 00020000:00000010:3.0:1773941319.301220:0:5433:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000002149e0f1. 00008000:00000010:3.0:1773941319.301221:0:5433:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000b3580e99. 00000080:00000010:3.0:1773941319.301223:0:5433:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 000000001e799344. 00000080:00000010:3.0:1773941319.301225:0:5433:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 000000002abe1461. 00000020:00000010:3.0:1773941319.301227:0:5433:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c6d68ffc. 00000020:00000010:3.0:1773941319.301229:0:5433:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000008e0b0091. 00000020:00000010:3.0:1773941319.301232:0:5433:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000000bf6bd2. 00000008:00000010:3.0:1773941319.301234:0:5433:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000c8661c5e. 00020000:00000010:3.0:1773941319.301236:0:5433:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 00000000948107ec. 00008000:00000010:3.0:1773941319.301238:0:5433:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 0000000088a31a61. 00000080:00000010:3.0:1773941319.301239:0:5433:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 000000007dc1df9a. 00000020:00000010:3.0:1773941319.301240:0:5433:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000b7fbafb1. 00000020:00000010:3.0:1773941319.301244:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 0000000007e527fd. 00000020:00000010:3.0:1773941319.301245:0:5433:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at 00000000ea415e53. 00000020:00000010:3.0:1773941319.301251:0:5433:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 000000003fb4ac52. 00000020:00000010:3.0:1773941319.301254:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 152 at 0000000007e527fd. 00000020:00000010:3.0:1773941319.301260:0:5433:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b7fbafb1. 00000020:00000010:3.0:1773941319.301263:0:5433:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000c6d68ffc. 00000040:00000010:3.0:1773941319.301266:0:5433:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000012c246e7. 00000020:00000010:3.0:1773941319.301279:0:5433:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000ad1d41ab. 00000020:00000010:3.0:1773941319.301292:0:5433:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000a7c527cb. 00040000:00000010:3.0:1773941319.301295:0:5433:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000005d563209. 00040000:00000010:3.0:1773941319.301299:0:5433:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000185a396b. 00800000:00000010:3.0:1773941319.301310:0:5433:0:(lmv_obd.c:433:lmv_add_target()) kmalloced '(tgt)': 368 at 00000000297f6636. 00020000:00000010:3.0:1773941319.301314:0:5433:0:(lu_tgt_descs.c:371:ltd_add_tgt()) kmalloced '(ltd->ltd_tgt_idx[index / (((1UL) << 12) / sizeof(void *))])': 4096 at 0000000062746648. 00000020:00000010:3.0:1773941319.301317:0:5433:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000c6d68ffc. 00000040:00000010:3.0:1773941319.301321:0:5433:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 0000000012c246e7. 00000020:00000010:3.0:1773941319.301323:0:5433:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 00000000ad1d41ab. 00000020:00000010:3.0:1773941319.301326:0:5433:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 00000000a7c527cb. 00040000:00000010:3.0:1773941319.301329:0:5433:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at 000000005d563209. 00040000:00000010:3.0:1773941319.301331:0:5433:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 00000000185a396b. 00000020:00000010:3.0:1773941319.301333:0:5433:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000b7fbafb1. 00000020:00000010:3.0:1773941319.301338:0:5433:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c33b6f03. 00000020:00000010:3.0:1773941319.301341:0:5433:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000008f1a1760. 00000020:00000010:3.0:1773941319.301343:0:5433:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000d6d5aaca. 00000008:00000010:3.0:1773941319.301346:0:5433:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 0000000074ea5404. 00020000:00000010:3.0:1773941319.301349:0:5433:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000002149e0f1. 00008000:00000010:3.0:1773941319.301353:0:5433:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000b3580e99. 00000080:00000010:3.0:1773941319.301356:0:5433:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000001e799344. 00000080:00000010:3.0:1773941319.301359:0:5433:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000002abe1461. 00000020:00000010:3.0:1773941319.301366:0:5433:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 000000008f1a1760. 00000020:00000010:3.0:1773941319.301369:0:5433:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000d6d5aaca. 00000008:00000010:3.0:1773941319.301372:0:5433:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 0000000074ea5404. 00020000:00000010:3.0:1773941319.301374:0:5433:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000002149e0f1. 00008000:00000010:3.0:1773941319.301376:0:5433:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000b3580e99. 00000080:00000010:3.0:1773941319.301378:0:5433:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 000000001e799344. 00000080:00000010:3.0:1773941319.301380:0:5433:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 000000002abe1461. 00000020:00000010:3.0:1773941319.301382:0:5433:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c33b6f03. 00000020:00000010:3.0:1773941319.301385:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 152 at 0000000007e527fd. 00000020:00000010:3.0:1773941319.301387:0:5433:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at 000000003fb4ac52. 00000020:00000010:3.0:1773941319.301392:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000005d563209. 00000020:00000010:3.0:1773941319.301395:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000005d563209. 00000020:00000010:3.0:1773941319.301407:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000005d563209. 00000020:00000010:3.0:1773941319.301410:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000005d563209. 00000020:00000010:3.0:1773941319.301416:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000005d563209. 00000020:00000010:3.0:1773941319.301418:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000005d563209. 00000020:00000010:3.0:1773941319.301422:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at 000000003fb4ac52. 00000100:00000010:3.0:1773941319.301425:0:5433:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at 00000000843ceaea. 00000400:00000200:3.0:1773941319.301427:0:5433:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1773941319.301430:0:5433:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.143@tcp 00000400:00000200:3.0:1773941319.301433:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.301435:0:5433:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.143@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1773941319.301439:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.301443:0:5433:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1773941319.301444:0:5433:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp: 0. pending discovery 00000400:00000200:3.0:1773941319.301447:0:5433:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.143@tcp 00000100:00000010:3.0:1773941319.301448:0:5433:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at 00000000843ceaea. 00000020:00000010:3.0:1773941319.301450:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 64 at 000000003fb4ac52. 00000020:00000010:3.0:1773941319.301453:0:5433:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 000000003fb4ac52. 00000020:00000010:3.0:1773941319.301456:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 0000000007e527fd. 00000020:00000010:3.0:1773941319.301461:0:5433:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at 0000000050e71b4d. 00000020:00000010:3.0:1773941319.301464:0:5433:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 00000000ad1d41ab. 00000020:00000010:3.0:1773941319.301469:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 0000000007e527fd. 00000020:00000010:3.0:1773941319.301471:0:5433:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at 000000003fb4ac52. 00000020:00000010:3.0:1773941319.301474:0:5433:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 000000003fb4ac52. 00000020:00000010:3.0:1773941319.301475:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 0000000007e527fd. 00000001:00000010:3.0:1773941319.301488:0:5433:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 000000005d563209. 00000001:00000010:3.0:1773941319.301490:0:5433:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 00000000fe26b691. 00000001:00000010:3.0:1773941319.301492:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000c33b6f03. 00000001:00000010:3.0:1773941319.301493:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000008f1a1760. 00000001:00000010:3.0:1773941319.301495:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000006c012289. 00000001:00000010:3.0:1773941319.301496:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000b19e1dd5. 00000020:00000010:3.0:1773941319.301498:0:5433:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004a57f713. 00000020:00000010:3.0:1773941319.301499:0:5433:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000008e0b0091. 00000020:00000010:3.0:1773941319.301507:0:5433:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at 0000000000bf6bd2. 00000008:00000010:3.0:1773941319.301509:0:5433:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000c8661c5e. 00020000:00000010:3.0:1773941319.301511:0:5433:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 00000000948107ec. 00008000:00000010:3.0:1773941319.301513:0:5433:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 0000000088a31a61. 00000080:00000010:3.0:1773941319.301516:0:5433:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 000000007dc1df9a. 00000020:00000010:3.0:1773941319.301521:0:5433:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000088149318. 00000020:00000010:3.0:1773941319.301524:0:5433:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000e38cedad. 00000020:00000010:3.0:1773941319.301527:0:5433:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000d6d5aaca. 00000008:00000010:3.0:1773941319.301529:0:5433:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 0000000074ea5404. 00020000:00000010:3.0:1773941319.301531:0:5433:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000002149e0f1. 00008000:00000010:3.0:1773941319.301533:0:5433:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000b3580e99. 00000080:00000010:3.0:1773941319.301536:0:5433:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000001e799344. 00000080:00000010:3.0:1773941319.301539:0:5433:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000002abe1461. 00000008:00000010:3.0:1773941319.301542:0:5433:0:(osc_dev.c:192:osc_device_alloc()) kmalloced '(osc)': 104 at 00000000c1b56566. 00010000:00000010:3.0:1773941319.301551:0:5433:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at 000000003b17baf8. 00010000:00000010:3.0:1773941319.301556:0:5433:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at 0000000082ba5342. 00000020:00000010:3.0:1773941319.301562:0:5433:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at 000000009e5aef4e. 00000100:00000200:3.0:1773941319.301572:0:5433:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.143@tcp->12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941319.301576:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.301581:0:5433:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.143@tcp primary NID 192.168.204.143@tcp rc 0 00010000:00000010:3.0:1773941319.301586:0:5433:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at 00000000d35ebd33. 00010000:00000010:3.0:1773941319.301589:0:5433:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at 0000000076bd5300. 00000001:00000010:3.0:1773941319.301592:0:5433:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at 00000000713b3fba. 00000001:00000010:3.0:1773941319.301595:0:5433:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 128 at 0000000047eb6dfc. 00000001:00000010:3.0:1773941319.301600:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000004cdb856e. 00000001:00000010:3.0:1773941319.301606:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000c194ec14. 00000001:00000010:3.0:1773941319.301620:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000001d0f82a9. 00000001:00000010:3.0:1773941319.301627:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000002ef47b0a. 00000001:00000010:3.0:1773941319.301633:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 0000000053a34604. 00000001:00000010:3.0:1773941319.301639:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000002037b555. 00000001:00000010:3.0:1773941319.301652:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000704f1cf8. 00000001:00000010:3.0:1773941319.301659:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000009dadfaf4. 00000001:00000010:3.0:1773941319.301669:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000008cc364f8. 00000001:00000010:3.0:1773941319.301675:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000f04467ac. 00000001:00000010:3.0:1773941319.301689:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000006fb0d087. 00000001:00000010:3.0:1773941319.301696:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000c8b17216. 00000001:00000010:3.0:1773941319.301732:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000e4c7a6cf. 00000001:00000010:3.0:1773941319.301739:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000001eeec82c. 00000001:00000010:3.0:1773941319.301753:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000127a45a3. 00000001:00000010:3.0:1773941319.301760:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000a7dd85fb. 00000020:00000010:3.0:1773941319.301852:0:5433:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000deae0c63. 00000020:00000010:3.0:1773941319.301857:0:5433:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at 000000007f63adf4. 00000020:00000010:3.0:1773941319.301937:0:5433:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 0000000067a73de5. 00000020:00000010:3.0:1773941319.301941:0:5433:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at 00000000f2f0d3fa. 00000020:00000010:3.0:1773941319.302355:0:5433:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 0000000004b1f048. 00000020:00000010:3.0:1773941319.302359:0:5433:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 000000005fd7ea9f. 00000100:00000010:3.0:1773941319.302391:0:5433:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000058e00681. 00000100:00000010:3.0:1773941319.302547:0:5433:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 00000000ba88f0fa. 00000100:00000010:3.0:1773941319.302552:0:5433:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f76430a4. 00000100:00000010:3.0:1773941319.302729:0:5433:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 0000000028d33fc8. 00000100:00000010:3.0:1773941319.302735:0:5433:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000be0746c4. 00000100:00000010:3.0:1773941319.302866:0:5433:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 00000000ef8171c0. 00000100:00000010:2.0:1773941319.302898:0:5433:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005825508a. 00000100:00000010:2.0:1773941319.303036:0:5433:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 0000000028bfe7c3. 00000100:00000010:2.0:1773941319.303040:0:5433:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000d9dd73d9. 00000100:00000010:2.0:1773941319.303198:0:5433:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 0000000088de238a. 00000100:00000010:2.0:1773941319.303203:0:5433:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000021b5861e. 00000100:00000010:2.0:1773941319.303339:0:5433:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 000000008e31e6f6. 00000100:00000010:2.0:1773941319.303342:0:5433:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000059feff12. 00000100:00000010:2.0:1773941319.303475:0:5433:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 00000000f3f0732b. 00000100:00000010:2.0:1773941319.303480:0:5433:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a5f2452c. 00000100:00000010:2.0:1773941319.303620:0:5433:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 00000000170b2ba9. 00000100:00000010:2.0:1773941319.303625:0:5433:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000046bb1d71. 00000100:00000010:2.0:1773941319.303825:0:5433:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 000000003485a383. 00000100:00000010:2.0:1773941319.303830:0:5433:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000737afe10. 00000100:00000010:2.0:1773941319.303981:0:5433:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 000000009921a543. 00000020:00000010:2.0:1773941319.303988:0:5433:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000e38cedad. 00000020:00000010:2.0:1773941319.303991:0:5433:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000d6d5aaca. 00000008:00000010:2.0:1773941319.303995:0:5433:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 0000000074ea5404. 00020000:00000010:2.0:1773941319.303998:0:5433:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000002149e0f1. 00008000:00000010:2.0:1773941319.304002:0:5433:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000b3580e99. 00000080:00000010:2.0:1773941319.304011:0:5433:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 000000001e799344. 00000080:00000010:2.0:1773941319.304015:0:5433:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 000000002abe1461. 00000020:00000010:2.0:1773941319.304018:0:5433:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000088149318. 00000020:00000010:2.0:1773941319.304022:0:5433:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000008e0b0091. 00000020:00000010:2.0:1773941319.304026:0:5433:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at 0000000000bf6bd2. 00000008:00000010:2.0:1773941319.304030:0:5433:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000c8661c5e. 00020000:00000010:2.0:1773941319.304034:0:5433:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 00000000948107ec. 00008000:00000010:2.0:1773941319.304036:0:5433:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 0000000088a31a61. 00000080:00000010:2.0:1773941319.304039:0:5433:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 000000007dc1df9a. 00000020:00000010:2.0:1773941319.304041:0:5433:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004a57f713. 00000020:00000010:2.0:1773941319.304045:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 0000000007e527fd. 00000020:00000010:2.0:1773941319.304048:0:5433:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at 000000003fb4ac52. 00000020:00000010:2.0:1773941319.304056:0:5433:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 000000005225bf4b. 00000020:00000010:2.0:1773941319.304061:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 120 at 00000000ce816243. 00000020:00000010:2.0:1773941319.304067:0:5433:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000007dee1ecb. 00000020:00000010:2.0:1773941319.304071:0:5433:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000060ef869a. 00000020:00000010:2.0:1773941319.304074:0:5433:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 000000004d585041. 00000008:00000010:2.0:1773941319.304079:0:5433:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 000000002e552e67. 00020000:00000010:2.0:1773941319.304082:0:5433:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 0000000095086b82. 00008000:00000010:2.0:1773941319.304085:0:5433:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000c7c6c10e. 00000080:00000010:2.0:1773941319.304089:0:5433:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000e3693c7a. 00000080:00000010:2.0:1773941319.304093:0:5433:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000005d6e4f50. 00020000:00000010:2.0:1773941319.304103:0:5433:0:(lov_obd.c:492:lov_add_target()) kmalloced '(tgt)': 368 at 00000000f7a554a1. 00020000:00000010:2.0:1773941319.304107:0:5433:0:(lu_tgt_descs.c:371:ltd_add_tgt()) kmalloced '(ltd->ltd_tgt_idx[index / (((1UL) << 12) / sizeof(void *))])': 4096 at 0000000061a128fd. 00020000:00000010:2.0:1773941319.304114:0:5433:0:(lov_dev.c:357:lov_expand_targets()) kmalloced '(newd)': 512 at 00000000eef58704. 00000020:00000010:2.0:1773941319.304118:0:5433:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 0000000060ef869a. 00000020:00000010:2.0:1773941319.304121:0:5433:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 000000004d585041. 00000008:00000010:2.0:1773941319.304123:0:5433:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 000000002e552e67. 00020000:00000010:2.0:1773941319.304127:0:5433:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 0000000095086b82. 00008000:00000010:2.0:1773941319.304129:0:5433:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000c7c6c10e. 00000080:00000010:2.0:1773941319.304132:0:5433:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 00000000e3693c7a. 00000080:00000010:2.0:1773941319.304134:0:5433:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 000000005d6e4f50. 00000020:00000010:2.0:1773941319.304143:0:5433:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000007dee1ecb. 00000020:00000010:2.0:1773941319.304147:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 120 at 00000000ce816243. 00000020:00000010:2.0:1773941319.304151:0:5433:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at 000000005225bf4b. 00000020:00000010:2.0:1773941319.304157:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000009a50adf6. 00000020:00000010:2.0:1773941319.304160:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000009a50adf6. 00000020:00000010:2.0:1773941319.304167:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000009a50adf6. 00000020:00000010:2.0:1773941319.304169:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000009a50adf6. 00000020:00000010:2.0:1773941319.304174:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000009a50adf6. 00000020:00000010:2.0:1773941319.304177:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000009a50adf6. 00000020:00000010:2.0:1773941319.304180:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at 000000005225bf4b. 00000100:00000010:2.0:1773941319.304184:0:5433:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at 0000000042318080. 00000400:00000200:2.0:1773941319.304200:0:5433:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:2.0:1773941319.304205:0:5433:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.143@tcp 00000400:00000200:2.0:1773941319.304210:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.304214:0:5433:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.143@tcp NID flags 0x100001: -17 00000400:00000200:2.0:1773941319.304219:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.304224:0:5433:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1773941319.304227:0:5433:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp: 0. pending discovery 00000400:00000200:2.0:1773941319.304230:0:5433:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.143@tcp 00000100:00000010:2.0:1773941319.304232:0:5433:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at 0000000042318080. 00000020:00000010:2.0:1773941319.304236:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 64 at 000000005225bf4b. 00000020:00000010:2.0:1773941319.304241:0:5433:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 000000005225bf4b. 00000020:00000010:2.0:1773941319.304244:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 0000000038da6382. 00000020:00000010:2.0:1773941319.304260:0:5433:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at 000000008318ca9e. 00000020:00000010:2.0:1773941319.304266:0:5433:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 000000000ad59d69. 00000020:00000010:2.0:1773941319.304275:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 0000000038da6382. 00000020:00000010:2.0:1773941319.304278:0:5433:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at 000000005225bf4b. 00000020:00000010:2.0:1773941319.304282:0:5433:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 000000005225bf4b. 00000020:00000010:2.0:1773941319.304285:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 0000000038da6382. 00000001:00000010:2.0:1773941319.304303:0:5433:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 000000009a50adf6. 00000001:00000010:2.0:1773941319.304307:0:5433:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 000000008abcb937. 00000001:00000010:2.0:1773941319.304310:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000b8de91f5. 00000001:00000010:2.0:1773941319.304318:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000bd4e1775. 00000001:00000010:2.0:1773941319.304320:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000007dee1ecb. 00000001:00000010:2.0:1773941319.304323:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 0000000060ef869a. 00000020:00000010:2.0:1773941319.304328:0:5433:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000072fc1062. 00000020:00000010:2.0:1773941319.304331:0:5433:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000ce816243. 00000020:00000010:2.0:1773941319.304341:0:5433:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000004a4d7546. 00000008:00000010:2.0:1773941319.304351:0:5433:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000facffa85. 00020000:00000010:2.0:1773941319.304362:0:5433:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 0000000014653895. 00008000:00000010:2.0:1773941319.304380:0:5433:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 00000000196b11de. 00000080:00000010:2.0:1773941319.304390:0:5433:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 00000000138cf6de. 00000020:00000010:2.0:1773941319.304394:0:5433:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000fc56c341. 00000020:00000010:2.0:1773941319.304397:0:5433:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000004a3a2562. 00000020:00000010:2.0:1773941319.304399:0:5433:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 000000004d585041. 00000008:00000010:2.0:1773941319.304402:0:5433:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 000000002e552e67. 00020000:00000010:2.0:1773941319.304404:0:5433:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 0000000095086b82. 00008000:00000010:2.0:1773941319.304406:0:5433:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000c7c6c10e. 00000080:00000010:2.0:1773941319.304408:0:5433:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000e3693c7a. 00000080:00000010:2.0:1773941319.304410:0:5433:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000005d6e4f50. 00000008:00000010:2.0:1773941319.304412:0:5433:0:(osc_dev.c:192:osc_device_alloc()) kmalloced '(osc)': 104 at 0000000067d08615. 00010000:00000010:2.0:1773941319.304421:0:5433:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at 00000000e2c0370b. 00010000:00000010:2.0:1773941319.304426:0:5433:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at 000000005bba7b07. 00000020:00000010:2.0:1773941319.304432:0:5433:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at 000000003fd639d4. 00000100:00000200:2.0:1773941319.304445:0:5433:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.143@tcp->12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941319.304450:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.304454:0:5433:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.143@tcp primary NID 192.168.204.143@tcp rc 0 00010000:00000010:2.0:1773941319.304458:0:5433:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at 0000000058da7f26. 00010000:00000010:2.0:1773941319.304463:0:5433:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at 00000000ca55d63f. 00000001:00000010:2.0:1773941319.304466:0:5433:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at 00000000027735e5. 00000001:00000010:2.0:1773941319.304470:0:5433:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 128 at 00000000bec6d642. 00000001:00000010:2.0:1773941319.304475:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000004fb3e54e. 00000001:00000010:2.0:1773941319.304481:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000009d03c983. 00000001:00000010:2.0:1773941319.304486:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000005d4eb82e. 00000001:00000010:2.0:1773941319.304497:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000003284a744. 00000001:00000010:2.0:1773941319.304503:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000d307cbb1. 00000001:00000010:2.0:1773941319.304509:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000f6c9e9fa. 00000001:00000010:2.0:1773941319.304514:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000a2871554. 00000001:00000010:2.0:1773941319.304526:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000049f7980. 00000001:00000010:2.0:1773941319.304533:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000d122be1a. 00000001:00000010:2.0:1773941319.304538:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 0000000069ab9183. 00000001:00000010:2.0:1773941319.304544:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000cc18e90c. 00000001:00000010:2.0:1773941319.304556:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 0000000024e7d448. 00000001:00000010:2.0:1773941319.304563:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000009b343df0. 00000001:00000010:2.0:1773941319.304568:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000eeb9cb86. 00000001:00000010:2.0:1773941319.304574:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 0000000005921821. 00000001:00000010:2.0:1773941319.304584:0:5433:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000d8aabc07. 00000020:00000010:2.0:1773941319.304630:0:5433:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000ca9857bf. 00000020:00000010:2.0:1773941319.304632:0:5433:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at 0000000014be13c0. 00000020:00000010:2.0:1773941319.304681:0:5433:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000bf78addf. 00000020:00000010:2.0:1773941319.304685:0:5433:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at 00000000a0dde6ea. 00000020:00000010:2.0:1773941319.304861:0:5433:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000b9ebb94a. 00000020:00000010:2.0:1773941319.304865:0:5433:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 00000000d8bc7886. 00000020:00000010:2.0:1773941319.304885:0:5433:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 000000004a3a2562. 00000020:00000010:2.0:1773941319.304888:0:5433:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 000000004d585041. 00000008:00000010:2.0:1773941319.304891:0:5433:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 000000002e552e67. 00020000:00000010:2.0:1773941319.304893:0:5433:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 0000000095086b82. 00008000:00000010:2.0:1773941319.304896:0:5433:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000c7c6c10e. 00000080:00000010:2.0:1773941319.304898:0:5433:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 00000000e3693c7a. 00000080:00000010:2.0:1773941319.304900:0:5433:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 000000005d6e4f50. 00000020:00000010:2.0:1773941319.304902:0:5433:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000fc56c341. 00000020:00000010:2.0:1773941319.304904:0:5433:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000ce816243. 00000020:00000010:2.0:1773941319.304907:0:5433:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000004a4d7546. 00000008:00000010:2.0:1773941319.304909:0:5433:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000facffa85. 00020000:00000010:2.0:1773941319.304911:0:5433:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 0000000014653895. 00008000:00000010:2.0:1773941319.304918:0:5433:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 00000000196b11de. 00000080:00000010:2.0:1773941319.304920:0:5433:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 00000000138cf6de. 00000020:00000010:2.0:1773941319.304921:0:5433:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000072fc1062. 00000020:00000010:2.0:1773941319.304923:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 0000000038da6382. 00000020:00000010:2.0:1773941319.304924:0:5433:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at 000000005225bf4b. 00000020:00000010:2.0:1773941319.304927:0:5433:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 000000005225bf4b. 00000020:00000010:2.0:1773941319.304929:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 120 at 00000000ce816243. 00000020:00000010:2.0:1773941319.304932:0:5433:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000003cb3fd1d. 00000020:00000010:2.0:1773941319.304933:0:5433:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000017f75b5e. 00000020:00000010:2.0:1773941319.304935:0:5433:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 000000004d585041. 00000008:00000010:2.0:1773941319.304937:0:5433:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 000000002e552e67. 00020000:00000010:2.0:1773941319.304938:0:5433:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 0000000095086b82. 00008000:00000010:2.0:1773941319.304939:0:5433:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000c7c6c10e. 00000080:00000010:2.0:1773941319.304941:0:5433:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000e3693c7a. 00000080:00000010:2.0:1773941319.304942:0:5433:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000005d6e4f50. 00020000:00000010:2.0:1773941319.304946:0:5433:0:(lov_obd.c:492:lov_add_target()) kmalloced '(tgt)': 368 at 000000001c62b821. 00000020:00000010:2.0:1773941319.304948:0:5433:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 0000000017f75b5e. 00000020:00000010:2.0:1773941319.304949:0:5433:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 000000004d585041. 00000008:00000010:2.0:1773941319.304951:0:5433:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 000000002e552e67. 00020000:00000010:2.0:1773941319.304952:0:5433:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 0000000095086b82. 00008000:00000010:2.0:1773941319.304953:0:5433:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000c7c6c10e. 00000080:00000010:2.0:1773941319.304954:0:5433:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 00000000e3693c7a. 00000080:00000010:2.0:1773941319.304955:0:5433:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 000000005d6e4f50. 00000020:00000010:2.0:1773941319.304956:0:5433:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000003cb3fd1d. 00000020:00000010:2.0:1773941319.304958:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 120 at 00000000ce816243. 00000020:00000010:2.0:1773941319.304959:0:5433:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at 000000005225bf4b. 00000020:00000010:2.0:1773941319.304962:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000baad2b1f. 00000020:00000010:2.0:1773941319.304963:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000baad2b1f. 00000020:00000010:2.0:1773941319.304965:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000baad2b1f. 00000020:00000010:2.0:1773941319.304966:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000baad2b1f. 00000020:00000010:2.0:1773941319.304967:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 56 at 000000005225bf4b. 00000020:00000010:2.0:1773941319.304968:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 56 at 000000005225bf4b. 00000100:00000010:2.0:1773941319.304977:0:5433:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000083d4724a. 02000000:00000010:2.0:1773941319.304985:0:5433:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 000000003cb3fd1d. 00000100:00000010:2.0:1773941319.304995:0:5433:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000038da6382. 02000000:00000010:2.0:1773941319.305014:0:5433:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at 00000000fb20f24f. 00000400:00000200:2.0:1773941319.305020:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941319.305022:0:5433:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000db9eafd5. 00000400:00000010:2.0:1773941319.305026:0:5433:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at 00000000a7599618. 00000100:00000200:2.0:1773941319.305032:0:5433:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112241132544, portal 25 00000400:00000010:2.0:1773941319.305035:0:5433:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000997d0be3. 00000100:00000200:2.0:1773941319.305037:0:5433:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112241132544, offset 0 00000400:00000200:2.0:1773941319.305044:0:5433:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941319.305049:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.305052:0:5433:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941319.305056:0:5433:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941319.305059:0:5433:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941319.305062:0:5433:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941319.305065:0:5433:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941319.305067:0:5433:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [11:11:256:4294967295] 192.168.204.143@tcp LPNI seq info [11:11:8:4294967295] 00000400:00000200:2.0:1773941319.305070:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.305077:0:5433:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941319.305081:0:5433:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941319.305084:0:5433:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000baad2b1f. 00000800:00000200:2.0:1773941319.305090:0:5433:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941319.305096:0:5433:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941319.305099:0:5433:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000baad2b1f type 1, nob 368 niov 1 nkiov 1 00000400:00000200:2.0:1773941319.305112:0:5433:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:2.0:1773941319.305114:0:5433:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000004d585041. 00000100:00000010:2.0:1773941319.305122:0:5433:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000263ebbd5. 00000800:00000200:3.0:1773941319.305203:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941319.305208:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000baad2b1f. 00000400:00000200:3.0:1773941319.305212:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.305218:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941319.305223:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000997d0be3 00000400:00000010:3.0:1773941319.305225:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000997d0be3. 00000100:00000200:3.0:1773941319.305234:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000083d4724a x1860112241132544/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941319.306227:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.306231:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941319.306236:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.306244:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.306247:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.306249:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 8472 into portal 25 MB=0x69bc321600400 00000400:00000200:2.0:1773941319.306253:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.306257:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 8472/8472 into md 0x24d [4] + 224 00000400:00000200:2.0:1773941319.306288:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941319.306293:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941319.306300:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000083d4724a x1860112241132544/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941319.306354:0:5433:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000db9eafd5. 00000400:00000200:3.0:1773941319.306360:0:5433:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000a7599618 00000400:00000010:3.0:1773941319.306362:0:5433:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at 00000000a7599618. 00000100:00000200:3.0:1773941319.306374:0:5433:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000083d4724a x1860112241132544/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941319.306382:0:5433:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000083d4724a x1860112241132544/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941319.306402:0:5433:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 0000000038da6382. 02000000:00000010:3.0:1773941319.306411:0:5433:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at 00000000fb20f24f. 02000000:00000010:3.0:1773941319.306421:0:5433:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 000000003cb3fd1d. 00000100:00000010:3.0:1773941319.306426:0:5433:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000083d4724a. 00000020:00000010:3.0:1773941319.306434:0:5433:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000d6d7e3a8. 00000020:00000010:3.0:1773941319.306437:0:5433:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000d6d7e3a8. 00000040:00000010:3.0:1773941319.306442:0:5433:0:(llog.c:891:llog_process_thread()) kvfree 'buf': 8192 at 0000000007f45625. 00000020:00000010:3.0:1773941319.306448:0:5433:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at 00000000b28c71cf. 00000020:00000010:3.0:1773941319.306458:0:5433:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000cc6806d0. 00000040:00000010:3.0:1773941319.306461:0:5433:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 00000000ceb8ec80. 00000020:00000010:3.0:1773941319.306464:0:5433:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 000000003a503b55. 00000020:00000010:3.0:1773941319.306467:0:5433:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at 000000007dc14f5e. 00000020:00000010:3.0:1773941319.306470:0:5433:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 0000000056cad018. 80000000:00000010:3.0:1773941319.306473:0:5433:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at 000000001a3861b4. 40000000:00000010:3.0:1773941319.306476:0:5433:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at 000000003e930e78. 00100000:00000010:3.0:1773941319.306479:0:5433:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at 00000000a96b4ca5. 00040000:00000010:3.0:1773941319.306482:0:5433:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at 000000009aea171e. 00040000:00000010:3.0:1773941319.306485:0:5433:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at 000000003074a38a. 00040000:00000010:3.0:1773941319.306487:0:5433:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 00000000c56bbe08. 00000004:00000010:3.0:1773941319.306491:0:5433:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at 000000006d5cee44. 00000004:00000010:3.0:1773941319.306494:0:5433:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at 000000001827d6c7. 00000004:00000010:3.0:1773941319.306498:0:5433:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at 000000006b6b69dc. 00002000:00000010:3.0:1773941319.306501:0:5433:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at 0000000073644ac5. 00000004:00000010:3.0:1773941319.306504:0:5433:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at 000000000fe7dd4f. 00000020:00000010:3.0:1773941319.306507:0:5433:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000355b1b86. 00000040:00000010:0.0:1773941319.306825:0:5414:0:(llog.c:986:llog_process_or_fork()) kfreed 'lpi': 96 at 00000000a35a07c3. 00000040:00000010:0.0:1773941319.306832:0:5414:0:(llog.c:70:llog_free_handle()) kvfree 'loghandle->lgh_hdr': 8192 at 00000000919865ef. 00000040:00000010:0.0:1773941319.306841:0:5414:0:(llog.c:72:llog_free_handle()) kfreed 'loghandle': 352 at 000000009eeedb85. 00000020:00000010:0.0:1773941319.306845:0:5414:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000f678b570. 00000040:00000010:0.0:1773941319.306849:0:5414:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 0000000031a659b1. 00000020:00000010:0.0:1773941319.306852:0:5414:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 00000000174ab7f0. 00000020:00000010:0.0:1773941319.306855:0:5414:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 00000000e27fc341. 80000000:00000010:0.0:1773941319.306858:0:5414:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at 000000008b1511c1. 00040000:00000010:0.0:1773941319.306862:0:5414:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 0000000004f4855c. 20000000:00000010:0.0:1773941319.306865:0:5414:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at 0000000007e25331. 00000020:00000010:0.0:1773941319.306868:0:5414:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000837136c8. 10000000:00000010:0.0:1773941319.306871:0:5414:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at 00000000ea630b28. 00000100:00000010:0.0:1773941319.306892:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005151efc1. 02000000:00000010:0.0:1773941319.306899:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 000000008b1a7ce8. 00010000:00000010:0.0:1773941319.306907:0:5414:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at 00000000d0031946. 00010000:00000010:0.0:1773941319.306911:0:5414:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at 00000000ba720e3d. 00000020:00000010:0.0:1773941319.306915:0:5414:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at 000000005ab7168d. 00000100:00000010:0.0:1773941319.306925:0:5414:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000bd183e77. 02000000:00000010:0.0:1773941319.306931:0:5414:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000d2b9fd8b. 00000400:00000200:0.0:1773941319.306935:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941319.306938:0:5414:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c405db14. 00000400:00000010:0.0:1773941319.306941:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000c9a4cf67. 00000100:00000200:0.0:1773941319.306943:0:5414:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241132672, portal 25 00000400:00000010:0.0:1773941319.306946:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000d80f83c8. 00000100:00000200:0.0:1773941319.306948:0:5414:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241132672, offset 0 00000400:00000200:0.0:1773941319.306953:0:5414:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941319.306957:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.306960:0:5414:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941319.306963:0:5414:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941319.306964:0:5414:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941319.306966:0:5414:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941319.306968:0:5414:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941319.306971:0:5414:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [12:12:256:4294967295] 192.168.204.143@tcp LPNI seq info [12:12:8:4294967295] 00000400:00000200:0.0:1773941319.306975:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.306988:0:5414:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941319.306992:0:5414:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941319.306994:0:5414:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000006f0853be. 00000800:00000200:0.0:1773941319.306998:0:5414:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941319.307003:0:5414:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941319.307005:0:5414:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000006f0853be type 1, nob 424 niov 1 nkiov 1 00000400:00000200:0.0:1773941319.307016:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941319.307057:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941319.307060:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006f0853be. 00000400:00000200:3.0:1773941319.307064:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.307067:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941319.307074:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000d80f83c8 00000400:00000010:3.0:1773941319.307075:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000d80f83c8. 00000100:00000200:3.0:1773941319.307078:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005151efc1 x1860112241132672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941319.307899:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.307903:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941319.307907:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.307909:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.307911:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.307914:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 344 into portal 25 MB=0x69bc321600480 00000400:00000200:2.0:1773941319.307917:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.307919:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 344/344 into md 0x25d [1] + 224 00000400:00000200:2.0:1773941319.307928:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941319.307931:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941319.307935:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005151efc1 x1860112241132672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941319.307961:0:5414:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c405db14. 00000400:00000200:0.0:1773941319.307965:0:5414:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c9a4cf67 00000400:00000010:0.0:1773941319.307967:0:5414:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000c9a4cf67. 00000100:00000200:0.0:1773941319.307972:0:5414:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005151efc1 x1860112241132672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941319.307979:0:5414:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005151efc1 x1860112241132672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1773941319.307990:0:5414:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000bd183e77. 02000000:00000010:0.0:1773941319.308000:0:5414:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000d2b9fd8b. 02000000:00000010:0.0:1773941319.308003:0:5414:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 000000008b1a7ce8. 00000100:00000010:0.0:1773941319.308005:0:5414:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005151efc1. 10000000:00000010:0.0:1773941319.308011:0:5414:0:(mgc_request.c:1545:mgc_process_recover_log()) kmalloced '(pages)': 2048 at 000000000769044b. 00000100:00000010:0.0:1773941319.308506:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005151efc1. 02000000:00000010:0.0:1773941319.308512:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 000000008b1a7ce8. 00000100:00000010:0.0:1773941319.308518:0:5414:0:(client.c:169:ptlrpc_new_bulk()) kmalloced '(desc)': 984 at 00000000d2b9fd8b. 00000100:00000010:0.0:1773941319.308521:0:5414:0:(client.c:174:ptlrpc_new_bulk()) kmalloced '(desc->bd_vec)': 4096 at 00000000bbeb2f42. 00000100:00000010:0.0:1773941319.308531:0:5414:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000bd183e77. 00000400:00000200:0.0:1773941319.308540:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941319.308542:0:5414:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c405db14. 00000400:00000010:0.0:1773941319.308548:0:5414:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 4216 at 00000000919865ef. 00000100:00000200:0.0:1773941319.308551:0:5414:0:(niobuf.c:453:ptlrpc_register_bulk()) Setup 1 bulk put-sink buffers: 256 pages 1048576 bytes, mbits x0x69bc321600500-0x69bc321600500, portal 33 02000000:00000010:0.0:1773941319.308556:0:5414:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000055fda90. 00000400:00000200:0.0:1773941319.308558:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941319.308565:0:5414:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b7f25210. 00000400:00000010:0.0:1773941319.308569:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000c9a4cf67. 00000100:00000200:0.0:1773941319.308571:0:5414:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241132800, portal 25 00000400:00000010:0.0:1773941319.308574:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000858ff6f7. 00000100:00000200:0.0:1773941319.308576:0:5414:0:(niobuf.c:94:ptl_send_buf()) Sending 304 bytes to portal 26, xid 1860112241132800, offset 0 00000400:00000200:0.0:1773941319.308581:0:5414:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941319.308585:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.308588:0:5414:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941319.308591:0:5414:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941319.308594:0:5414:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941319.308595:0:5414:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941319.308599:0:5414:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941319.308602:0:5414:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [13:13:256:4294967295] 192.168.204.143@tcp LPNI seq info [13:13:8:4294967295] 00000400:00000200:0.0:1773941319.308605:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.308612:0:5414:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941319.308615:0:5414:0:(socklnd_cb.c:1001:ksocknal_send()) sending 304 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941319.308618:0:5414:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000b7484b4a. 00000800:00000200:0.0:1773941319.308622:0:5414:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941319.308626:0:5414:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941319.308629:0:5414:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000b7484b4a type 1, nob 400 niov 1 nkiov 1 00000400:00000200:0.0:1773941319.308643:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:2.0:1773941319.308697:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941319.308700:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000b7484b4a. 00000400:00000200:2.0:1773941319.308722:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941319.308727:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941319.308729:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000858ff6f7 00000400:00000010:2.0:1773941319.308731:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000858ff6f7. 00000100:00000200:2.0:1773941319.308736:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005151efc1 x1860112241132800/t0(0) o256->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941335 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941319.309352:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.309359:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941319.309364:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.309368:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.309371:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.309374:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 132 into portal 33 MB=0x69bc321600500 00000400:00000200:2.0:1773941319.309378:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.309382:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 21 from 12345-192.168.204.143@tcp of length 132/132 into md 0x265 [256] + 0 00000400:00000010:2.0:1773941319.309385:0:2347:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000c405db14. 00000400:00000200:2.0:1773941319.309389:0:2347:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000919865ef 00000400:00000200:2.0:1773941319.309396:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941319.309400:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941319.309403:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000919865ef 00000400:00000010:2.0:1773941319.309405:0:2347:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 4216 at 00000000919865ef. 00000100:00000200:2.0:1773941319.309409:0:2347:0:(events.c:192:client_bulk_callback()) event type 2, status 0, req 000000005151efc1 desc 00000000d2b9fd8b mbits 1860112241132800 00000400:00000200:2.0:1773941319.309422:0:2347:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:2.0:1773941319.309425:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.309437:0:2347:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source Specified: 192.168.204.43@tcp to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941319.309442:0:2347:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [14:14:256:4294967295] 192.168.204.143@tcp LPNI seq info [14:14:8:4294967295] 00000400:00000200:2.0:1773941319.309446:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.309453:0:2347:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:192.168.204.43@tcp) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) 192.168.204.143@tcp : ACK try# 0 00000800:00000200:2.0:1773941319.309456:0:2347:0:(socklnd_cb.c:1001:ksocknal_send()) sending 0 bytes in 256 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941319.309460:0:2347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 4328 at 0000000075766f06. 00000800:00000200:2.0:1773941319.309463:0:2347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941319.309466:0:2347:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941319.309468:0:2347:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 0000000075766f06 type 0, nob 96 niov 1 nkiov 0 00000800:00000200:2.0:1773941319.309505:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941319.309508:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 4328 at 0000000075766f06. 00000400:00000200:2.0:1773941319.309511:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941319.309515:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: ACK: OK 00000400:00000200:2.0:1773941319.309973:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.309982:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941319.309987:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.309990:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.309992:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.309995:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 240 into portal 25 MB=0x69bc321600500 00000400:00000200:2.0:1773941319.309998:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.310001:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 240/240 into md 0x26d [1] + 224 00000400:00000200:2.0:1773941319.310009:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941319.310012:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941319.310017:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005151efc1 x1860112241132800/t0(0) o256->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941335 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941319.310052:0:5414:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b7f25210. 00000400:00000200:0.0:1773941319.310056:0:5414:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c9a4cf67 00000400:00000010:0.0:1773941319.310057:0:5414:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000c9a4cf67. 00000100:00000200:0.0:1773941319.310061:0:5414:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005151efc1 x1860112241132800/t0(0) o256->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941319.310067:0:5414:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005151efc1 x1860112241132800/t0(0) o256->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1773941319.310079:0:5414:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000bd183e77. 10000000:00000010:0.0:1773941319.310086:0:5414:0:(mgc_request.c:1297:mgc_apply_recover_logs()) kmalloced '(buf)': 4096 at 00000000457f75c2. 00000100:00000200:0.0:1773941319.310094:0:5414:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.204.143@tcp with 192.168.204.143@tcp 00000100:00000200:0.0:1773941319.310099:0:5414:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.204.143@tcp' 00000400:00000200:0.0:1773941319.310101:0:5414:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:0.0:1773941319.310105:0:5414:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.143@tcp 00000400:00000200:0.0:1773941319.310108:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.310112:0:5414:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.143@tcp NID flags 0x100001: -17 00000400:00000200:0.0:1773941319.310116:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.310120:0:5414:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941319.310123:0:5414:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp: 0. pending discovery 00000400:00000200:0.0:1773941319.310126:0:5414:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.143@tcp 10000000:00000010:0.0:1773941319.310131:0:5414:0:(mgc_request.c:1479:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at 00000000027ad767. 00000020:00000010:0.0:1773941319.310139:0:5414:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000008d17d6fe. 00000020:00000010:0.0:1773941319.310143:0:5414:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000005fc31d56. 00000020:00000010:0.0:1773941319.310146:0:5414:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000a35a07c3. 00000008:00000010:0.0:1773941319.310152:0:5414:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000156ded8b. 00020000:00000010:0.0:1773941319.310155:0:5414:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000007d1da2d6. 00008000:00000010:0.0:1773941319.310158:0:5414:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000dcba8250. 00000080:00000010:0.0:1773941319.310163:0:5414:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000001e445e5e. 00000080:00000010:0.0:1773941319.310165:0:5414:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000002e05162d. 00000020:00000010:0.0:1773941319.310209:0:5414:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 000000005fc31d56. 00000020:00000010:0.0:1773941319.310211:0:5414:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000a35a07c3. 00000008:00000010:0.0:1773941319.310214:0:5414:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000156ded8b. 00020000:00000010:0.0:1773941319.310216:0:5414:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000007d1da2d6. 00008000:00000010:0.0:1773941319.310218:0:5414:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000dcba8250. 00000080:00000010:0.0:1773941319.310221:0:5414:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 000000001e445e5e. 00000080:00000010:0.0:1773941319.310222:0:5414:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 000000002e05162d. 00000020:00000010:0.0:1773941319.310224:0:5414:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000008d17d6fe. 10000000:00000010:0.0:1773941319.310226:0:5414:0:(mgc_request.c:1495:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at 00000000027ad767. 00000100:00000200:0.0:1773941319.310232:0:5414:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.204.143@tcp with 192.168.204.143@tcp 00000100:00000200:0.0:1773941319.310233:0:5414:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.204.143@tcp' 00000400:00000200:0.0:1773941319.310235:0:5414:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:0.0:1773941319.310237:0:5414:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.143@tcp 00000400:00000200:0.0:1773941319.310239:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.310241:0:5414:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.143@tcp NID flags 0x100001: -17 00000400:00000200:0.0:1773941319.310243:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.310250:0:5414:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941319.310251:0:5414:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp: 0. pending discovery 00000400:00000200:0.0:1773941319.310253:0:5414:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.143@tcp 10000000:00000010:0.0:1773941319.310255:0:5414:0:(mgc_request.c:1479:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at 00000000027ad767. 00000020:00000010:0.0:1773941319.310258:0:5414:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000008d17d6fe. 00000020:00000010:0.0:1773941319.310259:0:5414:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000005fc31d56. 00000020:00000010:0.0:1773941319.310261:0:5414:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000a35a07c3. 00000008:00000010:0.0:1773941319.310263:0:5414:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000156ded8b. 00020000:00000010:0.0:1773941319.310270:0:5414:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000007d1da2d6. 00008000:00000010:0.0:1773941319.310273:0:5414:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000dcba8250. 00000080:00000010:0.0:1773941319.310275:0:5414:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000001e445e5e. 00000080:00000010:0.0:1773941319.310277:0:5414:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000002e05162d. 00000020:00000010:0.0:1773941319.310293:0:5414:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 000000005fc31d56. 00000020:00000010:0.0:1773941319.310295:0:5414:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000a35a07c3. 00000008:00000010:0.0:1773941319.310296:0:5414:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000156ded8b. 00020000:00000010:0.0:1773941319.310297:0:5414:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000007d1da2d6. 00008000:00000010:0.0:1773941319.310299:0:5414:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000dcba8250. 00000080:00000010:0.0:1773941319.310300:0:5414:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 000000001e445e5e. 00000080:00000010:0.0:1773941319.310301:0:5414:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 000000002e05162d. 00000020:00000010:0.0:1773941319.310303:0:5414:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000008d17d6fe. 10000000:00000010:0.0:1773941319.310306:0:5414:0:(mgc_request.c:1495:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at 00000000027ad767. 00000100:00000200:0.0:1773941319.310310:0:5414:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.204.143@tcp with 192.168.204.143@tcp 00000100:00000200:0.0:1773941319.310311:0:5414:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.204.143@tcp' 00000400:00000200:0.0:1773941319.310312:0:5414:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:0.0:1773941319.310314:0:5414:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.143@tcp 00000400:00000200:0.0:1773941319.310315:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.310317:0:5414:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.143@tcp NID flags 0x100001: -17 00000400:00000200:0.0:1773941319.310319:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.310321:0:5414:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941319.310322:0:5414:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp: 0. pending discovery 00000400:00000200:0.0:1773941319.310324:0:5414:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.143@tcp 10000000:00000010:0.0:1773941319.310326:0:5414:0:(mgc_request.c:1479:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at 00000000027ad767. 00000020:00000010:0.0:1773941319.310328:0:5414:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000008d17d6fe. 00000020:00000010:0.0:1773941319.310334:0:5414:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000005fc31d56. 00000020:00000010:0.0:1773941319.310336:0:5414:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000a35a07c3. 00000008:00000010:0.0:1773941319.310337:0:5414:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000156ded8b. 00020000:00000010:0.0:1773941319.310339:0:5414:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000007d1da2d6. 00008000:00000010:0.0:1773941319.310340:0:5414:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000dcba8250. 00000080:00000010:0.0:1773941319.310342:0:5414:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000001e445e5e. 00000080:00000010:0.0:1773941319.310344:0:5414:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000002e05162d. 00000020:00000010:0.0:1773941319.310358:0:5414:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 000000005fc31d56. 00000020:00000010:0.0:1773941319.310360:0:5414:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000a35a07c3. 00000008:00000010:0.0:1773941319.310361:0:5414:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000156ded8b. 00020000:00000010:0.0:1773941319.310362:0:5414:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000007d1da2d6. 00008000:00000010:0.0:1773941319.310364:0:5414:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000dcba8250. 00000080:00000010:0.0:1773941319.310366:0:5414:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 000000001e445e5e. 00000080:00000010:0.0:1773941319.310367:0:5414:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 000000002e05162d. 00000020:00000010:0.0:1773941319.310370:0:5414:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000008d17d6fe. 10000000:00000010:0.0:1773941319.310372:0:5414:0:(mgc_request.c:1495:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at 00000000027ad767. 10000000:00000010:0.0:1773941319.310374:0:5414:0:(mgc_request.c:1507:mgc_apply_recover_logs()) kfreed 'buf': 4096 at 00000000457f75c2. 02000000:00000010:0.0:1773941319.310379:0:5414:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000055fda90. 00000100:00000010:0.0:1773941319.310390:0:5414:0:(client.c:333:ptlrpc_free_bulk()) kvfree 'desc->bd_vec': 4096 at 00000000bbeb2f42. 00000100:00000010:0.0:1773941319.310393:0:5414:0:(client.c:334:ptlrpc_free_bulk()) kfreed 'desc': 984 at 00000000d2b9fd8b. 02000000:00000010:0.0:1773941319.310396:0:5414:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 000000008b1a7ce8. 00000100:00000010:0.0:1773941319.310399:0:5414:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005151efc1. 10000000:00000010:0.0:1773941319.310926:0:5414:0:(mgc_request.c:1663:mgc_process_recover_log()) kvfree 'pages': 2048 at 000000000769044b. 00000100:00000010:0.0:1773941319.310945:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005151efc1. 02000000:00000010:0.0:1773941319.310953:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 000000008b1a7ce8. 00010000:00000010:0.0:1773941319.310962:0:5414:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at 00000000e303a074. 00010000:00000010:0.0:1773941319.310966:0:5414:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at 0000000021acc175. 00000100:00000010:0.0:1773941319.310977:0:5414:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000bd183e77. 02000000:00000010:0.0:1773941319.310984:0:5414:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000d2b9fd8b. 00000400:00000200:0.0:1773941319.310988:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941319.310991:0:5414:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b7f25210. 00000400:00000010:0.0:1773941319.310994:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000c9a4cf67. 00000100:00000200:0.0:1773941319.310997:0:5414:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241132928, portal 25 00000400:00000010:0.0:1773941319.311006:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000f157f9a5. 00000100:00000200:0.0:1773941319.311008:0:5414:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241132928, offset 0 00000400:00000200:0.0:1773941319.311013:0:5414:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941319.311016:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.311019:0:5414:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941319.311022:0:5414:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941319.311024:0:5414:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941319.311026:0:5414:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941319.311028:0:5414:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941319.311031:0:5414:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [15:15:256:4294967295] 192.168.204.143@tcp LPNI seq info [15:15:8:4294967295] 00000400:00000200:0.0:1773941319.311034:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.311039:0:5414:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941319.311042:0:5414:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941319.311044:0:5414:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000444667dd. 00000800:00000200:0.0:1773941319.311047:0:5414:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941319.311051:0:5414:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941319.311053:0:5414:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000444667dd type 1, nob 424 niov 1 nkiov 1 00000400:00000200:0.0:1773941319.311064:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941319.311100:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941319.311103:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000444667dd. 00000400:00000200:3.0:1773941319.311107:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.311111:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941319.311113:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000f157f9a5 00000400:00000010:3.0:1773941319.311114:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000f157f9a5. 00000100:00000200:3.0:1773941319.311119:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005151efc1 x1860112241132928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941319.311789:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.311794:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941319.311798:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.311800:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.311802:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.311805:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 344 into portal 25 MB=0x69bc321600580 00000400:00000200:2.0:1773941319.311807:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.311810:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 344/344 into md 0x275 [1] + 224 00000400:00000200:2.0:1773941319.311817:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941319.311820:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941319.311825:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005151efc1 x1860112241132928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941319.312041:0:5414:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b7f25210. 00000400:00000200:0.0:1773941319.312047:0:5414:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c9a4cf67 00000400:00000010:0.0:1773941319.312049:0:5414:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000c9a4cf67. 00000100:00000200:0.0:1773941319.312055:0:5414:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005151efc1 x1860112241132928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941319.312064:0:5414:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005151efc1 x1860112241132928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1773941319.312079:0:5414:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000bd183e77. 02000000:00000010:0.0:1773941319.312091:0:5414:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000d2b9fd8b. 02000000:00000010:0.0:1773941319.312095:0:5414:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 000000008b1a7ce8. 00000100:00000010:0.0:1773941319.312098:0:5414:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005151efc1. 10000000:00000010:0.0:1773941319.312104:0:5414:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at 00000000b48a0940. 00000020:00000010:0.0:1773941319.312121:0:5414:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000008b1a7ce8. 00000020:00000010:0.0:1773941319.312129:0:5414:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000055fda90. 00000040:00000010:0.0:1773941319.312133:0:5414:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000008d17d6fe. 00000020:00000010:0.0:1773941319.312137:0:5414:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000000769044b. 00000020:00000010:0.0:1773941319.312142:0:5414:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000005fc31d56. 80000000:00000010:0.0:1773941319.312146:0:5414:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000a35a07c3. 00040000:00000010:0.0:1773941319.312150:0:5414:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000004d700348. 20000000:00000010:0.0:1773941319.312155:0:5414:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at 00000000c28d387a. 00000040:00000010:0.0:1773941319.312161:0:5414:0:(llog.c:42:llog_alloc_handle()) kmalloced '(loghandle)': 352 at 000000000cda8602. 00000100:00000010:0.0:1773941319.312166:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005151efc1. 02000000:00000010:0.0:1773941319.312173:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000cfcc3898. 00000100:00000010:0.0:1773941319.312181:0:5414:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000bd183e77. 02000000:00000010:0.0:1773941319.312203:0:5414:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000a43f78e9. 00000400:00000200:0.0:1773941319.312212:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941319.312215:0:5414:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b7f25210. 00000400:00000010:0.0:1773941319.312220:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000c9a4cf67. 00000100:00000200:0.0:1773941319.312223:0:5414:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241133056, portal 25 00000400:00000010:0.0:1773941319.312226:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004479030e. 00000100:00000200:0.0:1773941319.312229:0:5414:0:(niobuf.c:94:ptl_send_buf()) Sending 504 bytes to portal 26, xid 1860112241133056, offset 0 00000400:00000200:0.0:1773941319.312236:0:5414:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941319.312241:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.312245:0:5414:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941319.312249:0:5414:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941319.312252:0:5414:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941319.312255:0:5414:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941319.312258:0:5414:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941319.312261:0:5414:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [16:16:256:4294967295] 192.168.204.143@tcp LPNI seq info [16:16:8:4294967295] 00000400:00000200:0.0:1773941319.312266:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.312276:0:5414:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941319.312281:0:5414:0:(socklnd_cb.c:1001:ksocknal_send()) sending 504 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941319.312285:0:5414:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000063873a33. 00000800:00000200:0.0:1773941319.312289:0:5414:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941319.312300:0:5414:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941319.312303:0:5414:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 0000000063873a33 type 1, nob 600 niov 1 nkiov 1 00000400:00000200:0.0:1773941319.312322:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941319.312354:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941319.312357:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000063873a33. 00000400:00000200:3.0:1773941319.312360:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.312364:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941319.312366:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004479030e 00000400:00000010:3.0:1773941319.312367:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000004479030e. 00000100:00000200:3.0:1773941319.312371:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005151efc1 x1860112241133056/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941335 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941319.313011:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.313014:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941319.313017:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.313019:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.313021:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.313024:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 272 into portal 25 MB=0x69bc321600600 00000400:00000200:2.0:1773941319.313027:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.313034:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 272/272 into md 0x27d [1] + 224 00000400:00000200:2.0:1773941319.313050:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941319.313052:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941319.313057:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005151efc1 x1860112241133056/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941335 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941319.313106:0:5414:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000b7f25210. 00000400:00000200:1.0:1773941319.313111:0:5414:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c9a4cf67 00000400:00000010:1.0:1773941319.313113:0:5414:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000c9a4cf67. 00000100:00000200:1.0:1773941319.313119:0:5414:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005151efc1 x1860112241133056/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941319.313128:0:5414:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005151efc1 x1860112241133056/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941319.313142:0:5414:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000bd183e77. 02000000:00000010:1.0:1773941319.313149:0:5414:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000a43f78e9. 02000000:00000010:1.0:1773941319.313152:0:5414:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000cfcc3898. 00000100:00000010:1.0:1773941319.313154:0:5414:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005151efc1. 00000040:00000010:1.0:1773941319.313162:0:5414:0:(llog.c:412:llog_init_handle()) kmalloced '(llh)': 8192 at 0000000078a2f4e8. 00000100:00000010:1.0:1773941319.313168:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000002280d596. 02000000:00000010:1.0:1773941319.313175:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000a642d7eb. 00000100:00000010:1.0:1773941319.313181:0:5414:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000008d54acf. 02000000:00000010:1.0:1773941319.313211:0:5414:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at 000000007fe1b8f0. 00000400:00000200:1.0:1773941319.313217:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941319.313220:0:5414:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d265c9da. 00000400:00000010:1.0:1773941319.313225:0:5414:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at 0000000012d67e39. 00000100:00000200:1.0:1773941319.313228:0:5414:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112241133184, portal 25 00000400:00000010:1.0:1773941319.313233:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000c2d5c430. 00000100:00000200:1.0:1773941319.313242:0:5414:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112241133184, offset 0 00000400:00000200:1.0:1773941319.313248:0:5414:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941319.313253:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941319.313257:0:5414:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941319.313260:0:5414:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941319.313262:0:5414:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941319.313265:0:5414:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941319.313267:0:5414:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941319.313270:0:5414:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [17:17:256:4294967295] 192.168.204.143@tcp LPNI seq info [17:17:8:4294967295] 00000400:00000200:1.0:1773941319.313274:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941319.313282:0:5414:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941319.313286:0:5414:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941319.313290:0:5414:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000813b41cb. 00000800:00000200:1.0:1773941319.313294:0:5414:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941319.313299:0:5414:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941319.313301:0:5414:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000813b41cb type 1, nob 368 niov 1 nkiov 1 00000400:00000200:1.0:1773941319.313318:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941319.313351:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941319.313354:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000813b41cb. 00000400:00000200:3.0:1773941319.313358:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.313362:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941319.313365:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c2d5c430 00000400:00000010:3.0:1773941319.313366:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000c2d5c430. 00000100:00000200:3.0:1773941319.313372:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000002280d596 x1860112241133184/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941335 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941319.313955:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.313958:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941319.313962:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.313964:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.313966:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.313968:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 8416 into portal 25 MB=0x69bc321600680 00000400:00000200:2.0:1773941319.313970:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.313972:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 8416/8416 into md 0x285 [4] + 224 00000400:00000200:2.0:1773941319.313981:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941319.313983:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941319.313987:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000002280d596 x1860112241133184/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941335 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941319.314027:0:5414:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d265c9da. 00000400:00000200:1.0:1773941319.314032:0:5414:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000012d67e39 00000400:00000010:1.0:1773941319.314034:0:5414:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at 0000000012d67e39. 00000100:00000200:1.0:1773941319.314041:0:5414:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000002280d596 x1860112241133184/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941319.314050:0:5414:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000002280d596 x1860112241133184/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941319.314064:0:5414:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 0000000008d54acf. 02000000:00000010:1.0:1773941319.314072:0:5414:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at 000000007fe1b8f0. 02000000:00000010:1.0:1773941319.314081:0:5414:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000a642d7eb. 00000100:00000010:1.0:1773941319.314084:0:5414:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000002280d596. 00000040:00000010:1.0:1773941319.314092:0:5414:0:(llog.c:945:llog_process_or_fork()) kmalloced '(lpi)': 96 at 00000000349ca5ea. 00000020:00000010:1.0:1773941319.314246:0:5437:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a642d7eb. 00000020:00000010:1.0:1773941319.314253:0:5437:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000031f8007c. 00000040:00000010:1.0:1773941319.314264:0:5437:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000032868707. 00000020:00000010:1.0:1773941319.314269:0:5437:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000955c622a. 00000020:00000010:1.0:1773941319.314274:0:5437:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000056f1117. 00000020:00000010:1.0:1773941319.314279:0:5437:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000002da8b0eb. 80000000:00000010:1.0:1773941319.314283:0:5437:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000c033488b. 40000000:00000010:1.0:1773941319.314287:0:5437:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 000000003b3b8c80. 00100000:00000010:1.0:1773941319.314293:0:5437:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000174ab7f0. 00040000:00000010:1.0:1773941319.314298:0:5437:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 0000000089c67da8. 00040000:00000010:1.0:1773941319.314303:0:5437:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000009257afff. 00040000:00000010:1.0:1773941319.314306:0:5437:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000bd986ea5. 00000004:00000010:1.0:1773941319.314322:0:5437:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000000ff1e7c6. 00000004:00000010:1.0:1773941319.314329:0:5437:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 0000000096c5c78e. 00000004:00000010:1.0:1773941319.314335:0:5437:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 000000005e21086a. 00002000:00000010:1.0:1773941319.314340:0:5437:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 0000000045093ebe. 00000004:00000010:1.0:1773941319.314344:0:5437:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 000000004335bf11. 00000020:00000010:1.0:1773941319.314349:0:5437:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000006b3ca5fc. 00000040:00000010:1.0:1773941319.314356:0:5437:0:(llog.c:570:llog_process_thread()) kmalloced '(buf)': 8192 at 000000002b417db7. 00000100:00000010:1.0:1773941319.314362:0:5437:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000002280d596. 02000000:00000010:1.0:1773941319.314369:0:5437:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 000000005e701f10. 00000100:00000010:1.0:1773941319.314375:0:5437:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000012d67e39. 02000000:00000010:1.0:1773941319.314391:0:5437:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at 00000000ea87d886. 00000400:00000200:1.0:1773941319.314396:0:5437:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941319.314400:0:5437:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d265c9da. 00000400:00000010:1.0:1773941319.314405:0:5437:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at 000000005e0a7eaf. 00000100:00000200:1.0:1773941319.314407:0:5437:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112241133312, portal 25 00000400:00000010:1.0:1773941319.314412:0:5437:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000001b26a186. 00000100:00000200:1.0:1773941319.314414:0:5437:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112241133312, offset 0 00000400:00000200:1.0:1773941319.314420:0:5437:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941319.314424:0:5437:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941319.314428:0:5437:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941319.314431:0:5437:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941319.314434:0:5437:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941319.314437:0:5437:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941319.314445:0:5437:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941319.314448:0:5437:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [18:18:256:4294967295] 192.168.204.143@tcp LPNI seq info [18:18:8:4294967295] 00000400:00000200:1.0:1773941319.314452:0:5437:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941319.314461:0:5437:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941319.314466:0:5437:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941319.314470:0:5437:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000ca2490b1. 00000800:00000200:1.0:1773941319.314475:0:5437:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000007a1c9c6c] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941319.314479:0:5437:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941319.314482:0:5437:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000ca2490b1 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:1.0:1773941319.314497:0:5437:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941319.314533:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941319.314536:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000ca2490b1. 00000400:00000200:3.0:1773941319.314540:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.314545:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941319.314554:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000001b26a186 00000400:00000010:3.0:1773941319.314555:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000001b26a186. 00000100:00000200:3.0:1773941319.314561:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000002280d596 x1860112241133312/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941319.315106:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.315109:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941319.315113:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.315115:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.315117:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.315119:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 8472 into portal 25 MB=0x69bc321600700 00000400:00000200:2.0:1773941319.315121:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.315123:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 8472/8472 into md 0x28d [4] + 224 00000400:00000200:2.0:1773941319.315131:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941319.315134:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941319.315138:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000002280d596 x1860112241133312/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941319.315381:0:5437:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000d265c9da. 00000400:00000200:2.0:1773941319.315384:0:5437:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000005e0a7eaf 00000400:00000010:2.0:1773941319.315386:0:5437:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at 000000005e0a7eaf. 00000100:00000200:2.0:1773941319.315390:0:5437:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000002280d596 x1860112241133312/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941319.315395:0:5437:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000002280d596 x1860112241133312/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1773941319.315404:0:5437:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 0000000012d67e39. 02000000:00000010:2.0:1773941319.315408:0:5437:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at 00000000ea87d886. 02000000:00000010:2.0:1773941319.315416:0:5437:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 000000005e701f10. 00000100:00000010:2.0:1773941319.315419:0:5437:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000002280d596. 00000020:00000010:2.0:1773941319.315428:0:5437:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000baad2b1f. 00000020:00000010:2.0:1773941319.315431:0:5437:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000baad2b1f. 00000020:00000010:2.0:1773941319.315436:0:5437:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 28 at 00000000f131f9d9. 00000020:00000010:2.0:1773941319.315439:0:5437:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 112 at 0000000073eed90c. 00000020:00000010:2.0:1773941319.315457:0:5437:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 112 at ffff8b5b84f4e080. 00000020:00000010:2.0:1773941319.315459:0:5437:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 28 at ffff8b5b880dc860. 00000100:00000010:2.0:1773941319.315466:0:5437:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021d500. 02000000:00000010:2.0:1773941319.315471:0:5437:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b84dbae00. 00000100:00000010:2.0:1773941319.315476:0:5437:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b83f03000. 02000000:00000010:2.0:1773941319.315487:0:5437:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at ffff8b5ba0cc8000. 00000400:00000200:2.0:1773941319.315491:0:5437:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941319.315493:0:5437:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9c08. 00000400:00000010:2.0:1773941319.315497:0:5437:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at ffff8b5b83f03540. 00000100:00000200:2.0:1773941319.315500:0:5437:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112241133440, portal 25 00000400:00000010:2.0:1773941319.315505:0:5437:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34908. 00000100:00000200:2.0:1773941319.315506:0:5437:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112241133440, offset 0 00000400:00000200:2.0:1773941319.315510:0:5437:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941319.315514:0:5437:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.315516:0:5437:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941319.315524:0:5437:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941319.315526:0:5437:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941319.315528:0:5437:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941319.315530:0:5437:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941319.315532:0:5437:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [19:19:256:4294967295] 192.168.204.143@tcp LPNI seq info [19:19:8:4294967295] 00000400:00000200:2.0:1773941319.315536:0:5437:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.315540:0:5437:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941319.315543:0:5437:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941319.315546:0:5437:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bd7100. 00000800:00000200:2.0:1773941319.315549:0:5437:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941319.315552:0:5437:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941319.315554:0:5437:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bd7100 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:2.0:1773941319.315568:0:5437:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941319.315612:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941319.315616:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bd7100. 00000400:00000200:3.0:1773941319.315620:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.315625:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941319.315629:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34908 00000400:00000010:3.0:1773941319.315630:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34908. 00000100:00000200:3.0:1773941319.315637:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021d500 x1860112241133440/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941319.316126:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.316129:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941319.316133:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.316135:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.316137:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.316141:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 8472 into portal 25 MB=0x69bc321600780 00000400:00000200:2.0:1773941319.316143:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941319.316147:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 8472/8472 into md 0x295 [4] + 224 00000400:00000200:2.0:1773941319.316163:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941319.316166:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941319.316172:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021d500 x1860112241133440/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941319.316215:0:5437:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9c08. 00000400:00000200:0.0:1773941319.316220:0:5437:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f03540 00000400:00000010:0.0:1773941319.316221:0:5437:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at ffff8b5b83f03540. 00000100:00000200:0.0:1773941319.316227:0:5437:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021d500 x1860112241133440/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941319.316234:0:5437:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021d500 x1860112241133440/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941335 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1773941319.316245:0:5437:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b83f03000. 02000000:00000010:0.0:1773941319.316250:0:5437:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at ffff8b5ba0cc8000. 02000000:00000010:0.0:1773941319.316257:0:5437:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b84dbae00. 00000100:00000010:0.0:1773941319.316259:0:5437:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021d500. 00000020:00000010:0.0:1773941319.316265:0:5437:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff8b5ba0c22900. 00000020:00000010:0.0:1773941319.316267:0:5437:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff8b5ba0c22900. 00000040:00000010:0.0:1773941319.316270:0:5437:0:(llog.c:891:llog_process_thread()) kvfree 'buf': 8192 at ffff8b5b90d80000. 00000020:00000010:0.0:1773941319.316274:0:5437:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b82c592c0. 00000020:00000010:0.0:1773941319.316279:0:5437:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b902a1800. 00000040:00000010:0.0:1773941319.316282:0:5437:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b902a0c00. 00000020:00000010:0.0:1773941319.316284:0:5437:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90db2800. 00000020:00000010:0.0:1773941319.316287:0:5437:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b98a03250. 00000020:00000010:0.0:1773941319.316290:0:5437:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b902a1600. 80000000:00000010:0.0:1773941319.316292:0:5437:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f90c0. 40000000:00000010:0.0:1773941319.316295:0:5437:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b82c597c0. 00100000:00000010:0.0:1773941319.316298:0:5437:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b90db4000. 00040000:00000010:0.0:1773941319.316300:0:5437:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5ba2036900. 00040000:00000010:0.0:1773941319.316302:0:5437:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b855a8000. 00040000:00000010:0.0:1773941319.316305:0:5437:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b902a1e00. 00000004:00000010:0.0:1773941319.316307:0:5437:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b885e6000. 00000004:00000010:0.0:1773941319.316310:0:5437:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5b85e5e000. 00000004:00000010:0.0:1773941319.316313:0:5437:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b885e1800. 00002000:00000010:0.0:1773941319.316320:0:5437:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5b84c97c00. 00000004:00000010:0.0:1773941319.316323:0:5437:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5b84c94800. 00000020:00000010:0.0:1773941319.316325:0:5437:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b902a0200. 00000040:00000010:1.0:1773941319.316343:0:5414:0:(llog.c:986:llog_process_or_fork()) kfreed 'lpi': 96 at ffff8b5b880f98a0. 00000040:00000010:1.0:1773941319.316351:0:5414:0:(llog.c:70:llog_free_handle()) kvfree 'loghandle->lgh_hdr': 8192 at ffff8b5b90d82000. 00000040:00000010:1.0:1773941319.316355:0:5414:0:(llog.c:72:llog_free_handle()) kfreed 'loghandle': 352 at ffff8b5b89317200. 00000020:00000010:1.0:1773941319.316361:0:5414:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b89317c00. 00000040:00000010:1.0:1773941319.316365:0:5414:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b89317600. 00000020:00000010:1.0:1773941319.316369:0:5414:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5ba0b09800. 00000020:00000010:1.0:1773941319.316374:0:5414:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b89316600. 80000000:00000010:1.0:1773941319.316378:0:5414:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b85ef6a80. 00040000:00000010:1.0:1773941319.316381:0:5414:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b89317e00. 20000000:00000010:1.0:1773941319.316384:0:5414:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b89317000. 00000020:00000010:1.0:1773941319.316387:0:5414:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b89316000. 10000000:00000010:1.0:1773941319.316389:0:5414:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5ba039c280. 00000020:00000010:1.0:1773941319.316407:0:5414:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b839e2a00. 00000040:00000010:1.0:1773941319.316411:0:5414:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b839e3000. 00000020:00000010:1.0:1773941319.316415:0:5414:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90db1000. 00000020:00000010:1.0:1773941319.316419:0:5414:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b903b6200. 00040000:00000010:1.0:1773941319.316422:0:5414:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5ba2036e00. 00040000:00000010:1.0:1773941319.316426:0:5414:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b903b7200. 00000020:00000010:1.0:1773941319.316428:0:5414:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b839e3a00. 00000020:00000010:1.0:1773941319.316432:0:5414:0:(obd_mount.c:82:lustre_process_log()) kfreed 'lcfg': 184 at ffff8b5b990e6840. 00000020:00000010:1.0:1773941319.316435:0:5414:0:(obd_mount.c:84:lustre_process_log()) kfreed 'bufs': 104 at ffff8b5b90edb300. 00000080:00000010:1.0:1773941319.316442:0:5414:0:(llite_lib.c:1502:ll_fill_super()) kmalloced '(dt)': 33 at ffff8b5b82c59d80. 00000080:00000010:1.0:1773941319.316452:0:5414:0:(llite_lib.c:1508:ll_fill_super()) kmalloced '(md)': 33 at ffff8b5b82c59400. 00000080:00000010:1.0:1773941319.316459:0:5414:0:(llite_lib.c:348:client_common_fill_super()) kmalloced '(data)': 192 at ffff8b5b990e6840. 00000080:00000010:1.0:1773941319.316463:0:5414:0:(llite_lib.c:352:client_common_fill_super()) kmalloced '(osfs)': 144 at ffff8b5b990e63c0. 00000020:00000010:1.0:1773941319.316469:0:5414:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5b885e3800. 00000020:00000010:1.0:1773941319.316499:0:5414:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5b885e0000. 00000400:00000200:1.0:1773941319.316509:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941319.316514:0:5414:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000010:1.0:1773941319.316526:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61180. 02000000:00000010:1.0:1773941319.316534:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b84c94800. 40000000:00000010:1.0:1773941319.316569:0:5414:0:(fid_request.c:432:client_fid_init()) kmalloced '(cli->cl_seq)': 224 at ffff8b5ba2036e00. 40000000:00000010:1.0:1773941319.316573:0:5414:0:(fid_request.c:436:client_fid_init()) kmalloced '(prefix)': 133 at ffff8b5b990e60c0. 02000000:00000010:0.0:1773941319.316575:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0bf1400. 00000400:00000200:0.0:1773941319.316583:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941319.316587:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014b8f0. 00000400:00000010:0.0:1773941319.316592:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c440. 00000100:00000200:0.0:1773941319.316596:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241133568, portal 10 00000020:00000010:0.0:1773941319.316603:0:2351:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff8b5ba0b7c000. 00000400:00000010:0.0:1773941319.316607:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c908. 00000100:00000200:0.0:1773941319.316610:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1860112241133568, offset 0 00000400:00000200:0.0:1773941319.316615:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941319.316619:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.316623:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941319.316625:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941319.316628:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941319.316630:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941319.316633:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941319.316636:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [20:20:256:4294967295] 192.168.204.143@tcp LPNI seq info [20:20:8:4294967295] 00000400:00000200:0.0:1773941319.316639:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941319.316646:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941319.316650:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941319.316653:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5ba0c22900. 00000800:00000200:0.0:1773941319.316657:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 40000000:00000010:1.0:1773941319.316660:0:5414:0:(fid_request.c:444:client_fid_init()) kfreed 'prefix': 133 at ffff8b5b990e60c0. 00000800:00000200:0.0:1773941319.316663:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941319.316666:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5ba0c22900 type 1, nob 616 niov 1 nkiov 1 80000000:00000010:1.0:1773941319.316668:0:5414:0:(fld_request.c:150:fld_client_add_target()) kmalloced '(target)': 40 at ffff8b5b82c59f40. 00020000:00000010:1.0:1773941319.316676:0:5414:0:(lu_tgt_descs.c:102:lu_qos_add_tgt()) kmalloced '(svr)': 104 at ffff8b5b90edb300. 00000400:00000200:0.0:1773941319.316680:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941319.316686:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61c00. 02000000:00000010:1.0:1773941319.316693:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5ba2036200. 00000020:00000010:0.0:1773941319.316739:0:2353:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b9829f8c0. 00000800:00000200:3.0:1773941319.316745:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000020:00000010:0.0:1773941319.316745:0:2353:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b89317800. 80000000:00000010:0.0:1773941319.316749:0:2353:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b85ef60c0. 00000800:00000010:3.0:1773941319.316750:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5ba0c22900. 00800000:00000010:1.0:1773941319.316751:0:5414:0:(lmv_obd.c:1417:lmv_statfs()) kmalloced '(temp)': 144 at ffff8b5b990e60c0. 40000000:00000010:0.0:1773941319.316754:0:2353:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5ba039ce40. 00000400:00000200:3.0:1773941319.316758:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000010:1.0:1773941319.316759:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60e00. 00000400:00000200:3.0:1773941319.316766:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000008:00000010:0.0:1773941319.316767:0:2353:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5b89389650. 02000000:00000010:1.0:1773941319.316769:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5ba2036a00. 00000400:00000200:3.0:1773941319.316771:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c908 00020000:00000010:0.0:1773941319.316774:0:2353:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5ba0f7f0c0. 00000400:00000010:3.0:1773941319.316775:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c908. 00000100:00000010:1.0:1773941319.316776:0:5414:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6240. 00008000:00000010:0.0:1773941319.316782:0:2353:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b902811a8. 00000100:00000200:3.0:1773941319.316787:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61180 x1860112241133568/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941329 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00100000:00000010:0.0:1773941319.316789:0:2353:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5ba0b0b800. 00040000:00000010:0.0:1773941319.316795:0:2353:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5ba0c22d00. 00000020:00000010:1.0:1773941319.316801:0:2352:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b98a05a90. 00040000:00000010:0.0:1773941319.316801:0:2353:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5ba0b08000. 00000020:00000010:1.0:1773941319.316808:0:2352:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b902a0e00. 00040000:00000010:0.0:1773941319.316809:0:2353:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b89316800. 80000000:00000010:1.0:1773941319.316814:0:2352:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f9b40. 00000004:00000010:0.0:1773941319.316816:0:2353:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5ba0b08800. 40000000:00000010:1.0:1773941319.316819:0:2352:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b82c59780. 00000004:00000010:0.0:1773941319.316822:0:2353:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b98ac9000. 00000008:00000010:1.0:1773941319.316825:0:2352:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5b90d8d940. 00000004:00000010:0.0:1773941319.316828:0:2353:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5ba0b09000. 00020000:00000010:1.0:1773941319.316831:0:2352:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a10de40. 00002000:00000010:0.0:1773941319.316832:0:2353:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0bf3800. 00008000:00000010:1.0:1773941319.316838:0:2352:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d24e20. 00000004:00000010:0.0:1773941319.316839:0:2353:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5ba0bf0400. 00100000:00000010:1.0:1773941319.316846:0:2352:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b885e0800. 00000080:00000010:0.0:1773941319.316846:0:2353:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b902a41f8. 00040000:00000010:1.0:1773941319.316854:0:2352:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5ba2036000. 00000080:00000010:0.0:1773941319.316854:0:2353:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b8502db60. 00040000:00000010:1.0:1773941319.316863:0:2352:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b885e1000. 00000008:00000010:0.0:1773941319.316863:0:2353:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5b88e854a0. 00020000:00000010:0.0:1773941319.316868:0:2353:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5b85039440. 00040000:00000010:1.0:1773941319.316870:0:2352:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b902a0400. 00008000:00000010:0.0:1773941319.316872:0:2353:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5b902a33e0. 00000004:00000010:1.0:1773941319.316876:0:2352:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b885e4000. 00000080:00000010:0.0:1773941319.316877:0:2353:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b902a2f00. 00000004:00000010:1.0:1773941319.316888:0:2352:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b85e59000. 00000004:00000010:1.0:1773941319.316892:0:2352:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b885e2800. 00002000:00000010:1.0:1773941319.316895:0:2352:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b84c97c00. 00000004:00000010:1.0:1773941319.316898:0:2352:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5b84c94000. 00000080:00000010:1.0:1773941319.316902:0:2352:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b83f2e2a0. 00000080:00000010:1.0:1773941319.316906:0:2352:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b98a0e6c8. 00000008:00000010:1.0:1773941319.316918:0:2352:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5ba0b98dc0. 00020000:00000010:1.0:1773941319.316929:0:2352:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5ba0b9cfc0. 00008000:00000010:1.0:1773941319.316949:0:2352:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5ba0b14250. 00000080:00000010:1.0:1773941319.316961:0:2352:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5ba0bb53c0. 00000400:00000200:3.0:1773941319.319331:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.319338:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941319.319345:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.319348:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.319352:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.319356:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 10 MB=0x69bc321600800 00000400:00000200:3.0:1773941319.319361:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941319.319365:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.143@tcp of length 416/416 into md 0x2a5 [1] + 0 00000400:00000200:3.0:1773941319.319377:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941319.319381:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941319.319389:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61180 x1860112241133568/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941329 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941319.319427:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014b8f0. 00000400:00000200:3.0:1773941319.319432:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c440 00000400:00000010:3.0:1773941319.319433:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c440. 00000100:00000200:3.0:1773941319.319438:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61180 x1860112241133568/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941329 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941319.319444:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61180 x1860112241133568/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941329 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1773941319.319455:0:2351:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff8b5b85056000. 00000020:00000010:2.0:1773941319.320094:0:5439:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at ffff8b5b845ad340. 00000020:00000010:2.0:1773941319.320099:0:5439:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at ffff8b5b845ad340. 00000100:00000010:2.0:1773941319.320773:0:758:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8b5b9a59bea0. 00000020:00000010:0.0:1773941319.320982:0:5440:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at ffff8b5ba039c380. 00000020:00000010:0.0:1773941319.320988:0:5440:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at ffff8b5ba039c380. 00000020:00000010:1.0:1773941319.323250:0:5441:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at ffff8b5b82c59180. 00000020:00000010:1.0:1773941319.323255:0:5441:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at ffff8b5b82c59180. 00000100:00000010:3.0:1773941319.328832:0:33:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8b5b9a59bc60. 02000000:00000010:0.0:1773941320.319225:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0bf1400. 02000000:00000010:0.0:1773941320.319236:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b84c94800. 00000100:00000010:0.0:1773941320.319242:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61180. 02000000:00000010:0.0:1773941320.319265:0:2353:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0bf1400. 02000000:00000010:2.0:1773941320.319270:0:5414:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b83f3b400. 00000400:00000200:0.0:1773941320.319274:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941320.319281:0:2353:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014b000. 00000400:00000200:2.0:1773941320.319284:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941320.319288:0:5414:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9c08. 00000400:00000010:0.0:1773941320.319290:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c5d8. 00000400:00000010:2.0:1773941320.319293:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f345d8. 00000100:00000200:2.0:1773941320.319298:0:5414:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241133824, portal 10 00000100:00000200:0.0:1773941320.319298:0:2353:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241133696, portal 10 00000400:00000010:0.0:1773941320.319307:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6cb28. 00000100:00000200:0.0:1773941320.319313:0:2353:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1860112241133696, offset 0 00000020:00000010:2.0:1773941320.319315:0:5414:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff8b5b98af8000. 00000400:00000010:2.0:1773941320.319321:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34b28. 00000400:00000200:0.0:1773941320.319322:0:2353:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000100:00000200:2.0:1773941320.319324:0:5414:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1860112241133824, offset 0 00000400:00000200:2.0:1773941320.319330:0:5414:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941320.319333:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941320.319337:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941320.319341:0:2353:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941320.319342:0:5414:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941320.319347:0:5414:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941320.319351:0:5414:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941320.319354:0:2353:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941320.319356:0:5414:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941320.319361:0:5414:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941320.319361:0:2353:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941320.319369:0:2353:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941320.319370:0:5414:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [21:21:256:4294967295] 192.168.204.143@tcp LPNI seq info [21:21:8:4294967295] 00000400:00000200:2.0:1773941320.319376:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941320.319376:0:2353:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941320.319384:0:2353:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [22:22:256:4294967295] 192.168.204.143@tcp LPNI seq info [22:22:8:4294967295] 00000400:00000200:2.0:1773941320.319391:0:5414:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000400:00000200:0.0:1773941320.319395:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:2.0:1773941320.319399:0:5414:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941320.319403:0:5414:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bd7100. 00000400:00000200:0.0:1773941320.319408:0:2353:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941320.319414:0:5414:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941320.319420:0:2353:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941320.319424:0:2353:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5ba0c22500. 00000800:00000200:0.0:1773941320.319428:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941320.319449:0:2353:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941320.319458:0:2353:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5ba0c22500 type 1, nob 320 niov 1 nkiov 1 00000800:00000200:2.0:1773941320.319471:0:5414:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000400:00000200:0.0:1773941320.319474:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:2.0:1773941320.319476:0:5414:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bd7100 type 1, nob 320 niov 1 nkiov 1 00000100:00000010:0.0:1773941320.319480:0:2353:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b5b85ef6660. 00000400:00000200:2.0:1773941320.319484:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:0.0:1773941320.319492:0:2353:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b5b85ef64e0. 00000800:00000200:3.0:1773941320.319534:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941320.319538:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5ba0c22500. 00000400:00000200:3.0:1773941320.319542:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941320.319547:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941320.319551:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6cb28 00000400:00000010:3.0:1773941320.319553:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6cb28. 00000100:00000200:3.0:1773941320.319559:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61c00 x1860112241133696/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941336 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1773941320.319631:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941320.319634:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bd7100. 00000400:00000200:3.0:1773941320.319638:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941320.319641:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941320.319644:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34b28 00000400:00000010:3.0:1773941320.319645:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34b28. 00000100:00000200:3.0:1773941320.319650:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60e00 x1860112241133824/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941336 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941320.320621:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.320625:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941320.320628:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.320631:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.320639:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.320643:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 368 into portal 10 MB=0x69bc321600900 00000400:00000200:3.0:1773941320.320647:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.320692:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.143@tcp of length 368/368 into md 0x2b5 [1] + 224 00000400:00000200:3.0:1773941320.320729:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941320.320734:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941320.320742:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60e00 x1860112241133824/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941336 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941320.320769:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.320772:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941320.320775:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.320777:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.320780:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.320782:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 368 into portal 10 MB=0x69bc321600880 00000400:00000200:3.0:1773941320.320785:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.320787:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.143@tcp of length 368/368 into md 0x2ad [1] + 224 00000400:00000200:3.0:1773941320.320793:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941320.320795:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941320.320799:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61c00 x1860112241133696/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941336 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941320.320806:0:5414:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9c08. 00000400:00000200:2.0:1773941320.320811:0:5414:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f345d8 00000400:00000010:2.0:1773941320.320813:0:5414:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f345d8. 00000100:00000200:2.0:1773941320.320825:0:5414:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60e00 x1860112241133824/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941336 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941320.320836:0:5414:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60e00 x1860112241133824/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941336 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941320.320848:0:2353:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014b000. 00000400:00000200:0.0:1773941320.320854:0:2353:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c5d8 00000400:00000010:0.0:1773941320.320858:0:2353:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c5d8. 00000100:00000200:0.0:1773941320.320866:0:2353:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61c00 x1860112241133696/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941336 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1773941320.320878:0:5414:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6240. 00000100:00000200:0.0:1773941320.320878:0:2353:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61c00 x1860112241133696/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941336 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1773941320.320888:0:5414:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b83f3b400. 02000000:00000010:2.0:1773941320.320894:0:5414:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5ba2036a00. 00000100:00000010:2.0:1773941320.320898:0:5414:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60e00. 00800000:00000010:2.0:1773941320.320904:0:5414:0:(lmv_obd.c:1477:lmv_statfs()) kfreed 'temp': 144 at ffff8b5b990e60c0. 02000000:00000010:0.0:1773941320.320905:0:2353:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0bf1400. 02000000:00000010:0.0:1773941320.320911:0:2353:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5ba2036200. 00000100:00000010:2.0:1773941320.320916:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021df80. 00000100:00000010:0.0:1773941320.320916:0:2353:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61c00. 02000000:00000010:2.0:1773941320.320930:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b84dba600. 00000100:00000010:2.0:1773941320.320942:0:5414:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b83f03b40. 02000000:00000010:2.0:1773941320.320952:0:5414:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b83f3b400. 00000400:00000200:2.0:1773941320.320956:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941320.320958:0:5414:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9c08. 00000400:00000010:2.0:1773941320.320962:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f345d8. 00000100:00000200:2.0:1773941320.320964:0:5414:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241133952, portal 10 00000400:00000010:2.0:1773941320.320968:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f342a8. 00000100:00000200:2.0:1773941320.320970:0:5414:0:(niobuf.c:94:ptl_send_buf()) Sending 440 bytes to portal 12, xid 1860112241133952, offset 0 00000400:00000200:2.0:1773941320.320976:0:5414:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941320.320980:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941320.320984:0:5414:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941320.320987:0:5414:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941320.320989:0:5414:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941320.320991:0:5414:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941320.320994:0:5414:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941320.320997:0:5414:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [23:23:256:4294967295] 192.168.204.143@tcp LPNI seq info [23:23:8:4294967295] 00000400:00000200:2.0:1773941320.321001:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941320.321007:0:5414:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941320.321011:0:5414:0:(socklnd_cb.c:1001:ksocknal_send()) sending 440 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941320.321015:0:5414:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bd7100. 00000800:00000200:2.0:1773941320.321019:0:5414:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941320.321024:0:5414:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941320.321027:0:5414:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bd7100 type 1, nob 536 niov 1 nkiov 1 00000400:00000200:2.0:1773941320.321048:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941320.321114:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941320.321118:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bd7100. 00000400:00000200:3.0:1773941320.321122:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941320.321126:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941320.321128:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f342a8 00000400:00000010:3.0:1773941320.321130:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f342a8. 00000100:00000200:3.0:1773941320.321136:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021df80 x1860112241133952/t0(0) o40->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941336 ref 3 fl Rpc:r/209/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941320.321677:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.321680:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941320.321688:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.321689:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.321691:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.321693:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 448 into portal 10 MB=0x69bc321600980 00000400:00000200:3.0:1773941320.321695:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.321698:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.143@tcp of length 448/448 into md 0x2c5 [1] + 224 00000400:00000200:3.0:1773941320.321718:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941320.321722:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941320.321729:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021df80 x1860112241133952/t0(0) o40->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941336 ref 2 fl Rpc:Qr/209/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941320.321772:0:5414:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9c08. 00000400:00000200:2.0:1773941320.321775:0:5414:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f345d8 00000400:00000010:2.0:1773941320.321776:0:5414:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f345d8. 00000100:00000200:2.0:1773941320.321780:0:5414:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021df80 x1860112241133952/t0(0) o40->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941336 ref 2 fl Rpc:RQ/209/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941320.321786:0:5414:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021df80 x1860112241133952/t0(0) o40->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941336 ref 2 fl Rpc:RQU/209/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1773941320.321796:0:5414:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b83f03b40. 00000002:00000200:2.0:1773941320.321799:0:5414:0:(mdc_request.c:134:mdc_get_root()) root fid=[0x200000007:0x1:0x0], last_committed=0 02000000:00000010:2.0:1773941320.321801:0:5414:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b83f3b400. 02000000:00000010:2.0:1773941320.321804:0:5414:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b84dba600. 00000100:00000010:2.0:1773941320.321805:0:5414:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021df80. 00000020:00000010:2.0:1773941320.321819:0:5414:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5b90f26000. 00000020:00000010:2.0:1773941320.321845:0:5414:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5b90f21800. 00000400:00000200:2.0:1773941320.321853:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941320.321856:0:5414:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000010:2.0:1773941320.321865:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021df80. 02000000:00000010:2.0:1773941320.321870:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b83f3b400. 00000020:00000010:2.0:1773941320.321912:0:5414:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5b90f20000. 00000100:00000010:3.0:1773941320.321922:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90274a80. 00000400:00000200:2.0:1773941320.321923:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941320.321928:0:5414:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 02000000:00000010:0.0:1773941320.321929:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0bf1400. 02000000:00000010:3.0:1773941320.321932:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b98a69f00. 00000100:00000010:2.0:1773941320.321935:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021d180. 00000400:00000200:0.0:1773941320.321940:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 02000000:00000010:2.0:1773941320.321943:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b83f38000. 00000400:00000010:0.0:1773941320.321945:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014b000. 00000400:00000010:0.0:1773941320.321951:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c5d8. 00000100:00000200:0.0:1773941320.321956:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241134080, portal 4 00000020:00000010:3.0:1773941320.321959:0:2355:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b8849d588. 00000020:00000010:3.0:1773941320.321965:0:2355:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b893ce800. 80000000:00000010:3.0:1773941320.321970:0:2355:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc94e0. 00000020:00000010:0.0:1773941320.321970:0:2351:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff8b5ba0b7e000. 00000080:00000010:2.0:1773941320.321973:0:5414:0:(llite_lib.c:738:client_common_fill_super()) kmalloced '(op_data)': 384 at ffff8b5b84dba600. 40000000:00000010:3.0:1773941320.321978:0:2355:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b88f93300. 00000020:00000010:2.0:1773941320.321979:0:5414:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffff8b5b83f3b000. 00000400:00000010:0.0:1773941320.321980:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c2a8. 00000008:00000010:3.0:1773941320.321984:0:2355:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f0ef90. 00000020:00000010:2.0:1773941320.321986:0:5414:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffff8b5b83f38c00. 00000100:00000200:0.0:1773941320.321986:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1860112241134080, offset 0 00020000:00000010:3.0:1773941320.321989:0:2355:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b89382600. 00000100:00000010:2.0:1773941320.321995:0:5414:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021dc00. 00008000:00000010:3.0:1773941320.321996:0:2355:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b98a1ea60. 00000400:00000200:0.0:1773941320.321997:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00100000:00000010:3.0:1773941320.322003:0:2355:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b90269800. 02000000:00000010:2.0:1773941320.322004:0:5414:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b84dba400. 00000400:00000200:0.0:1773941320.322004:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00040000:00000010:3.0:1773941320.322009:0:2355:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98a69200. 00000100:00000010:2.0:1773941320.322013:0:5414:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b83f03b40. 00000400:00000200:0.0:1773941320.322013:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00040000:00000010:3.0:1773941320.322015:0:2355:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b9026b800. 00000400:00000200:0.0:1773941320.322019:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00040000:00000010:3.0:1773941320.322021:0:2355:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b893cf200. 00000400:00000200:0.0:1773941320.322022:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941320.322025:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000004:00000010:3.0:1773941320.322026:0:2355:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b90268000. 00000400:00000200:0.0:1773941320.322027:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941320.322031:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [24:24:256:4294967295] 192.168.204.143@tcp LPNI seq info [24:24:8:4294967295] 00000004:00000010:3.0:1773941320.322032:0:2355:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b90f30000. 00000400:00000200:0.0:1773941320.322036:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000004:00000010:3.0:1773941320.322037:0:2355:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b90268800. 00002000:00000010:3.0:1773941320.322041:0:2355:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b90261000. 00000400:00000200:0.0:1773941320.322044:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000004:00000010:3.0:1773941320.322046:0:2355:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5b90262800. 00000800:00000200:0.0:1773941320.322050:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000080:00000010:3.0:1773941320.322051:0:2355:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b98a56348. 00000800:00000010:0.0:1773941320.322054:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5ba0c22400. 00000080:00000010:3.0:1773941320.322055:0:2355:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b89396da0. 02000000:00000010:2.0:1773941320.322058:0:5414:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 131072 at ffff8b5b90d40000. 00000800:00000200:0.0:1773941320.322059:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000008:00000010:3.0:1773941320.322061:0:2355:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5ba0c322c0. 00000800:00000200:0.0:1773941320.322066:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000400:00000200:2.0:1773941320.322070:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:0.0:1773941320.322071:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5ba0c22400 type 1, nob 616 niov 1 nkiov 1 00020000:00000010:3.0:1773941320.322072:0:2355:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5ba0c37cc0. 00000400:00000010:2.0:1773941320.322074:0:5414:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9c08. 00008000:00000010:3.0:1773941320.322079:0:2355:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5b98a6b368. 00000400:00000010:2.0:1773941320.322080:0:5414:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 632 at ffff8b5b83f3ac00. 00000080:00000010:3.0:1773941320.322084:0:2355:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b98a6ce10. 00000100:00000200:2.0:1773941320.322085:0:5414:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 131072 bytes, xid 1860112241134464, portal 10 00000400:00000200:0.0:1773941320.322085:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941320.322092:0:5414:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f345d8. 02000000:00000010:3.0:1773941320.322094:0:2355:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b893cf800. 00000100:00000200:2.0:1773941320.322095:0:5414:0:(niobuf.c:94:ptl_send_buf()) Sending 448 bytes to portal 23, xid 1860112241134464, offset 0 00000400:00000200:3.0:1773941320.322101:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 02000000:00000010:0.0:1773941320.322104:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0bf1000. 00000400:00000200:2.0:1773941320.322105:0:5414:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941320.322106:0:2355:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f872c0. 00000400:00000200:0.0:1773941320.322111:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941320.322114:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941320.322116:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014b5d8. 00000400:00000010:3.0:1773941320.322121:0:2355:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a990. 00000400:00000200:2.0:1773941320.322122:0:5414:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000010:0.0:1773941320.322122:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c110. 00000100:00000200:3.0:1773941320.322128:0:2355:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241134208, portal 25 00000100:00000200:0.0:1773941320.322129:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241134336, portal 4 00000400:00000200:2.0:1773941320.322131:0:5414:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000010:3.0:1773941320.322134:0:2355:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a880. 00000020:00000010:0.0:1773941320.322136:0:2351:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff8b5ba0b78000. 00000400:00000200:2.0:1773941320.322137:0:5414:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000100:00000200:3.0:1773941320.322138:0:2355:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241134208, offset 0 00000400:00000010:0.0:1773941320.322142:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c000. 00000400:00000200:2.0:1773941320.322144:0:5414:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000100:00000200:0.0:1773941320.322146:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1860112241134336, offset 0 00000400:00000200:3.0:1773941320.322147:0:2355:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941320.322152:0:5414:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941320.322160:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941320.322161:0:5414:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [25:25:255:4294967295] 192.168.204.143@tcp LPNI seq info [25:25:7:4294967295] 00000400:00000200:2.0:1773941320.322181:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941320.322182:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941320.322191:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941320.322197:0:5414:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000400:00000200:0.0:1773941320.322199:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000200:2.0:1773941320.322205:0:5414:0:(socklnd_cb.c:1001:ksocknal_send()) sending 448 bytes in 1 frags to 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941320.322208:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000010:2.0:1773941320.322211:0:5414:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bd7100. 00000400:00000200:0.0:1773941320.322212:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941320.322219:0:2355:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941320.322219:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000800:00000200:2.0:1773941320.322221:0:5414:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000400:00000200:3.0:1773941320.322227:0:2355:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941320.322228:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000800:00000200:2.0:1773941320.322233:0:5414:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000400:00000200:3.0:1773941320.322236:0:2355:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941320.322237:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [26:26:254:4294967295] 192.168.204.143@tcp LPNI seq info [26:26:6:4294967295] 00000800:00000200:2.0:1773941320.322239:0:5414:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bd7100 type 1, nob 544 niov 1 nkiov 1 00000400:00000200:3.0:1773941320.322244:0:2355:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941320.322246:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.322251:0:2355:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941320.322251:0:5414:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.322258:0:2355:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [27:27:254:4294967295] 192.168.204.143@tcp LPNI seq info [27:27:6:4294967295] 00000400:00000200:3.0:1773941320.322263:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.322270:0:2355:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:3.0:1773941320.322276:0:2355:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941320.322281:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000010:3.0:1773941320.322282:0:2355:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69700. 00000800:00000200:0.0:1773941320.322287:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000200:3.0:1773941320.322291:0:2355:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000010:0.0:1773941320.322291:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5ba0c22800. 00000800:00000200:0.0:1773941320.322297:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:3.0:1773941320.322302:0:2355:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941320.322305:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:3.0:1773941320.322307:0:2355:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69700 type 1, nob 320 niov 1 nkiov 1 00000800:00000200:0.0:1773941320.322309:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5ba0c22800 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:3.0:1773941320.322320:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941320.322320:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000010:3.0:1773941320.322341:0:2354:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b8849c168. 00000020:00000010:3.0:1773941320.322345:0:2354:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b893cf600. 80000000:00000010:3.0:1773941320.322347:0:2354:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9960. 40000000:00000010:3.0:1773941320.322350:0:2354:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b88f93b40. 00000008:00000010:3.0:1773941320.322352:0:2354:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f0b7c8. 00020000:00000010:3.0:1773941320.322360:0:2354:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b89382d80. 00008000:00000010:3.0:1773941320.322363:0:2354:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b98a1ce20. 00100000:00000010:3.0:1773941320.322366:0:2354:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b90269000. 00040000:00000010:3.0:1773941320.322369:0:2354:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98a69000. 00040000:00000010:3.0:1773941320.322371:0:2354:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b9026c000. 00040000:00000010:3.0:1773941320.322373:0:2354:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b893ce200. 00000004:00000010:3.0:1773941320.322375:0:2354:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b9026a800. 00000004:00000010:3.0:1773941320.322378:0:2354:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b89208000. 00000004:00000010:3.0:1773941320.322380:0:2354:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b9026b000. 00002000:00000010:3.0:1773941320.322381:0:2354:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b90262400. 00000004:00000010:3.0:1773941320.322383:0:2354:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5b90260000. 00000080:00000010:3.0:1773941320.322385:0:2354:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b98a56b28. 00000080:00000010:3.0:1773941320.322387:0:2354:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b89396238. 00000008:00000010:3.0:1773941320.322395:0:2354:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5ba0c326e0. 00020000:00000010:3.0:1773941320.322397:0:2354:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5ba0c35440. 00008000:00000010:3.0:1773941320.322399:0:2354:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5b98a6ba90. 00000080:00000010:3.0:1773941320.322401:0:2354:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b98a6cf00. 00000800:00000200:3.0:1773941320.322416:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941320.322418:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5ba0c22400. 00000400:00000200:3.0:1773941320.322421:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941320.322424:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941320.322426:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c2a8 00000400:00000010:3.0:1773941320.322427:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c2a8. 00000100:00000200:3.0:1773941320.322431:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021df80 x1860112241134080/t0(0) o8->lustre-OST0000-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1773941320.322440:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941320.322441:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bd7100. 00000400:00000200:3.0:1773941320.322443:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941320.322444:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941320.322446:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f345d8 00000400:00000010:3.0:1773941320.322447:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f345d8. 00000100:00000200:3.0:1773941320.322449:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021dc00 x1860112241134464/t0(0) o33->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941336 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:0 00000800:00000200:3.0:1773941320.322474:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941320.322475:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69700. 00000400:00000200:3.0:1773941320.322477:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941320.322479:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941320.322481:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a880 00000400:00000010:3.0:1773941320.322482:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a880. 00000100:00000200:3.0:1773941320.322485:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90274a80 x1860112241134208/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941336 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1773941320.322511:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941320.322513:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5ba0c22800. 00000400:00000200:3.0:1773941320.322515:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941320.322517:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941320.322518:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c000 00000400:00000010:3.0:1773941320.322519:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c000. 00000100:00000200:3.0:1773941320.322522:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021d180 x1860112241134336/t0(0) o8->lustre-OST0001-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941320.323243:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.323248:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941320.323253:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.323256:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.323259:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.323262:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321600a80 00000400:00000200:3.0:1773941320.323266:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.323270:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x2ed [1] + 224 00000400:00000200:3.0:1773941320.323279:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941320.323282:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941320.323289:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90274a80 x1860112241134208/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941336 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941320.323338:0:2355:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f872c0. 00000400:00000200:2.0:1773941320.323342:0:2355:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a990 00000400:00000010:2.0:1773941320.323344:0:2355:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a990. 00000100:00000200:2.0:1773941320.323349:0:2355:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90274a80 x1860112241134208/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941336 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941320.323356:0:2355:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90274a80 x1860112241134208/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941336 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941320.323366:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 02000000:00000010:2.0:1773941320.323369:0:2355:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b893cf800. 02000000:00000010:2.0:1773941320.323372:0:2355:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b98a69f00. 00000400:00000200:3.0:1773941320.323377:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000100:00000010:2.0:1773941320.323383:0:2355:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90274a80. 00000400:00000200:3.0:1773941320.323387:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.323391:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.323395:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.323398:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 4 MB=0x69bc321600a00 00000400:00000200:3.0:1773941320.323402:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.323405:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.143@tcp of length 416/416 into md 0x2d5 [1] + 0 00000400:00000200:3.0:1773941320.323445:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941320.323450:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941320.323456:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021df80 x1860112241134080/t0(0) o8->lustre-OST0000-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941330 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941320.323505:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014b000. 00000400:00000200:0.0:1773941320.323508:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c5d8 00000400:00000010:0.0:1773941320.323510:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c5d8. 00000100:00000200:0.0:1773941320.323514:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021df80 x1860112241134080/t0(0) o8->lustre-OST0000-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941330 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941320.323520:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021df80 x1860112241134080/t0(0) o8->lustre-OST0000-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941330 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1773941320.323553:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0bf1400. 02000000:00000010:0.0:1773941320.323557:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b83f3b400. 00000100:00000010:0.0:1773941320.323560:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021df80. 00000400:00000200:3.0:1773941320.324022:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.324030:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941320.324033:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.324035:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.324037:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.324040:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 4 MB=0x69bc321600b00 00000400:00000200:3.0:1773941320.324042:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.324044:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.143@tcp of length 416/416 into md 0x2f5 [1] + 0 00000400:00000200:3.0:1773941320.324050:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941320.324053:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941320.324057:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021d180 x1860112241134336/t0(0) o8->lustre-OST0001-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941330 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941320.324098:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014b5d8. 00000400:00000200:0.0:1773941320.324101:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c110 00000400:00000010:0.0:1773941320.324102:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c110. 00000100:00000200:0.0:1773941320.324106:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021d180 x1860112241134336/t0(0) o8->lustre-OST0001-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941330 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941320.324112:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021d180 x1860112241134336/t0(0) o8->lustre-OST0001-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941330 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1773941320.324130:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0bf1000. 02000000:00000010:0.0:1773941320.324136:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b83f38000. 00000100:00000010:0.0:1773941320.324138:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021d180. 00000400:00000200:3.0:1773941320.335409:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.335444:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941320.335451:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.335455:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.335458:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.335461:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 464 into portal 10 MB=0x69bc321600b80 00000400:00000200:3.0:1773941320.335465:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941320.335479:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.143@tcp of length 464/464 into md 0x2dd [32] + 224 00000400:00000200:3.0:1773941320.335496:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941320.335499:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941320.335511:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021dc00 x1860112241134464/t0(0) o33->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941336 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:0 00000400:00000010:2.0:1773941320.335576:0:5414:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9c08. 00000400:00000200:2.0:1773941320.335582:0:5414:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f3ac00 00000400:00000010:2.0:1773941320.335583:0:5414:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 632 at ffff8b5b83f3ac00. 00000100:00000200:2.0:1773941320.335589:0:5414:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021dc00 x1860112241134464/t0(0) o33->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941336 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:0 00000100:00000200:2.0:1773941320.335596:0:5414:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021dc00 x1860112241134464/t0(0) o33->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941336 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:0 00000100:00000010:2.0:1773941320.335618:0:5414:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b83f03b40. 00000002:00000200:2.0:1773941320.335623:0:5414:0:(mdc_request.c:172:mdc_getattr_common()) mode: 40755 00000080:00000010:2.0:1773941320.335627:0:5414:0:(llite_lib.c:753:client_common_fill_super()) kfreed 'op_data': 384 at ffff8b5b84dba600. 00000020:00000010:2.0:1773941320.335646:0:5414:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8b5b8509c2d0. 00000020:00000010:2.0:1773941320.335649:0:5414:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84dba600. 00000020:00000010:2.0:1773941320.335653:0:5414:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b84dbac00. 00000020:00000010:2.0:1773941320.335655:0:5414:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5b9a59bea0. 00000008:00000010:2.0:1773941320.335660:0:5414:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f02ca0. 00020000:00000010:2.0:1773941320.335663:0:5414:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a2fd900. 00008000:00000010:2.0:1773941320.335666:0:5414:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d75c40. 00000080:00000010:2.0:1773941320.335670:0:5414:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b85eec348. 00000080:00000010:2.0:1773941320.335673:0:5414:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b98a88248. 00000020:00000010:2.0:1773941320.335675:0:5414:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84dbb600. 00000008:00000010:2.0:1773941320.335678:0:5414:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5b885b29a0. 00020000:00000010:2.0:1773941320.335680:0:5414:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5b84cbfa80. 00008000:00000010:2.0:1773941320.335682:0:5414:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5b82c37248. 00000080:00000010:2.0:1773941320.335684:0:5414:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b845f5690. 00000080:00000010:2.0:1773941320.335688:0:5414:0:(vvp_dev.c:174:vvp_device_alloc()) kmalloced '(vdv)': 72 at ffff8b5b9a59b480. 00000080:00000010:2.0:1773941320.335691:0:5414:0:(vvp_dev.c:182:vvp_device_alloc()) kmalloced '(site)': 576 at ffff8b5b83f3ac00. 00000020:00000010:2.0:1773941320.335739:0:5414:0:(lu_object.c:1154:lu_site_init()) kmalloced '(s->ls_bkts)': 18432 at ffff8b5b82c70000. 00000020:00000010:2.0:1773941320.335747:0:5414:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5b83f3bc00. 00000020:00000010:2.0:1773941320.335749:0:5414:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 192 at ffff8b5b83f03b40. 00000008:00000010:2.0:1773941320.335754:0:5414:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f06f90. 00020000:00000010:2.0:1773941320.335756:0:5414:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a2fd780. 00008000:00000010:2.0:1773941320.335761:0:5414:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d75530. 00000080:00000010:2.0:1773941320.335763:0:5414:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b85eecb28. 00000080:00000010:2.0:1773941320.335765:0:5414:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b98a88b68. 00000008:00000010:2.0:1773941320.335767:0:5414:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5b885b3e40. 00020000:00000010:2.0:1773941320.335768:0:5414:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5b84cbc240. 00008000:00000010:2.0:1773941320.335770:0:5414:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5b82c379c0. 00000080:00000010:2.0:1773941320.335771:0:5414:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b845f55a0. 00000008:00000010:2.0:1773941320.335774:0:5414:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f037c8. 00020000:00000010:2.0:1773941320.335775:0:5414:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a2fd3c0. 00008000:00000010:2.0:1773941320.335777:0:5414:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d74a98. 00000080:00000010:2.0:1773941320.335778:0:5414:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b85eec150. 00000080:00000010:2.0:1773941320.335779:0:5414:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b98a8afe8. 00000008:00000010:2.0:1773941320.335781:0:5414:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5b885b3b80. 00020000:00000010:2.0:1773941320.335783:0:5414:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5b84cbc480. 00008000:00000010:2.0:1773941320.335784:0:5414:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5b82c37dc0. 00000080:00000010:2.0:1773941320.335785:0:5414:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b845f5c30. 00000008:00000010:2.0:1773941320.335787:0:5414:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f06468. 00020000:00000010:2.0:1773941320.335788:0:5414:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a2fd600. 00008000:00000010:2.0:1773941320.335790:0:5414:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d75fc8. 00000080:00000010:2.0:1773941320.335791:0:5414:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b85eec5e8. 00000080:00000010:2.0:1773941320.335792:0:5414:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b98a88920. 00000008:00000010:2.0:1773941320.335794:0:5414:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5b885b3760. 00020000:00000010:2.0:1773941320.335795:0:5414:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5b84cbe640. 00008000:00000010:2.0:1773941320.335796:0:5414:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5b82c37128. 00000080:00000010:2.0:1773941320.335798:0:5414:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b845f5d20. 00000008:00000010:2.0:1773941320.335800:0:5414:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f05940. 00020000:00000010:2.0:1773941320.335801:0:5414:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a2fdd80. 00008000:00000010:2.0:1773941320.335803:0:5414:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d74000. 00000080:00000010:2.0:1773941320.335805:0:5414:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b85eeca80. 00000080:00000010:2.0:1773941320.335806:0:5414:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b98a8b908. 00000008:00000010:2.0:1773941320.335807:0:5414:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5b885b2840. 00020000:00000010:2.0:1773941320.335809:0:5414:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5b84cbd8c0. 00008000:00000010:2.0:1773941320.335821:0:5414:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5b82c373f0. 00000080:00000010:2.0:1773941320.335823:0:5414:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b845f5780. 00020000:00000010:2.0:1773941320.335829:0:5414:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffff8b5b9a59b720. 00020000:00000010:2.0:1773941320.335833:0:5414:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffff8b5b9a59b2a0. 00020000:00000010:2.0:1773941320.335836:0:5414:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffff8b5b9a59b4e0. 00000020:00000010:2.0:1773941320.335839:0:5414:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b84dbac00. 00000020:00000010:2.0:1773941320.335842:0:5414:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5b9a59bea0. 00000008:00000010:2.0:1773941320.335844:0:5414:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5ba0f02ca0. 00020000:00000010:2.0:1773941320.335847:0:5414:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a2fd900. 00008000:00000010:2.0:1773941320.335849:0:5414:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d75c40. 00000080:00000010:2.0:1773941320.335851:0:5414:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8b5b85eec348. 00000080:00000010:2.0:1773941320.335853:0:5414:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8b5b98a88248. 00000020:00000010:2.0:1773941320.335855:0:5414:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84dba600. 00000008:00000010:2.0:1773941320.335856:0:5414:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8b5b885b29a0. 00020000:00000010:2.0:1773941320.335858:0:5414:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5b84cbfa80. 00008000:00000010:2.0:1773941320.335860:0:5414:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8b5b82c37248. 00000080:00000010:2.0:1773941320.335862:0:5414:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8b5b845f5690. 00000020:00000010:2.0:1773941320.335863:0:5414:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84dbb600. 00000020:00000010:2.0:1773941320.335865:0:5414:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8b5b8509c2d0. 00000080:00000010:2.0:1773941320.335890:0:5414:0:(super25.c:43:ll_alloc_inode()) slab-alloced 'lli': 2072 at ffff8b5b83538840. 02000000:00000010:2.0:1773941320.335906:0:5414:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 131072 at ffff8b5b90d40000. 02000000:00000010:2.0:1773941320.335921:0:5414:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b84dba400. 00000100:00000010:2.0:1773941320.335924:0:5414:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021dc00. 00000080:00000010:2.0:1773941320.335931:0:5414:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff8b5b880dcfa0. 00000080:00000010:2.0:1773941320.335937:0:5414:0:(llite_lib.c:842:client_common_fill_super()) kfreed 'data': 192 at ffff8b5b990e6840. 00000080:00000010:2.0:1773941320.335941:0:5414:0:(llite_lib.c:843:client_common_fill_super()) kfreed 'osfs': 144 at ffff8b5b990e63c0. 00000080:00000010:2.0:1773941320.335966:0:5414:0:(llite_lib.c:1521:ll_fill_super()) kfreed 'md': 33 at ffff8b5b82c59400. 00000080:00000010:2.0:1773941320.335969:0:5414:0:(llite_lib.c:1523:ll_fill_super()) kfreed 'dt': 33 at ffff8b5b82c59d80. 00000080:00000010:2.0:1773941320.335972:0:5414:0:(llite_lib.c:1531:ll_fill_super()) kfreed 'cfg': 88 at ffff8b5b880f9a80. 00000080:02000400:2.0:1773941320.335975:0:5414:0:(llite_lib.c:1537:ll_fill_super()) Mounted lustre-client 00000020:00000010:0.0:1773941320.352742:0:267:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 448 at ffff8b5b89317a00. 00000080:00200008:2.0:1773941320.387902:0:5458:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b5b83538908), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1773941320.387911:0:5458:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908),name=/ 00000002:00010000:2.0:1773941320.387926:0:5458:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1773941320.387977:0:5458:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 5458 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941320.387985:0:5458:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 5458 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941320.387992:0:5458:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 5458 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1773941320.388010:0:5458:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021dc00 pname:cluuid:pid:xid:nid:opc:job mkdir:67bbba49-47e8-4446-a6da-0acb748c20a3:5458:1860112241134592:192.168.204.143@tcp:101: 00000100:00100000:2.0:1773941320.388069:0:5458:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b83f03180 going to sleep for 11 seconds 00000100:00100000:2.0:1773941320.389308:0:5458:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021dc00 pname:cluuid:pid:xid:nid:opc:job mkdir:67bbba49-47e8-4446-a6da-0acb748c20a3:5458:1860112241134592:192.168.204.143@tcp:101: 00010000:00010000:2.0:1773941320.389323:0:5458:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941320.389335:0:5458:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941320.389346:0:5458:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941320.389360:0:5458:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941320.389370:0:5458:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941320.389378:0:5458:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941320.389397:0:5458:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1773941320.389409:0:5458:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b5b9021dc00 x1860112241134592/t0(0) o101->lustre-MDT0000-mdc-ffff8b5b855a9800@192.168.204.143@tcp:12/10 lens 576/688 e 0 to 0 dl 1773941336 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'' uid:0 gid:0 projid:0 00000002:00010000:2.0:1773941320.389421:0:5458:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941320.389430:0:5458:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b5ba0c60c00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1773941320.389435:0:5458:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1773941320.389455:0:5458:0:(llite_lib.c:2979:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:2.0:1773941320.389460:0:5458:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) for lock 0xa33a5be14eae1bb8 00000080:00010000:2.0:1773941320.389465:0:5458:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) 00000080:00010000:2.0:1773941320.389468:0:5458:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xa33a5be14eae1bb8 from it ffffa10c434b3cb8 00010000:00010000:2.0:1773941320.389471:0:5458:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941320.389480:0:5458:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1773941321.375867:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:3.0:1773941321.750449:0:5510:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b5b83538908), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1773941321.750463:0:5510:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908),name=/ 00000002:00010000:3.0:1773941321.750487:0:5510:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1773941321.750504:0:5510:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1773941321.750520:0:5510:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1773941321.750528:0:5510:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1773941321.750535:0:5510:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) 00000080:00010000:3.0:1773941321.750538:0:5510:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) for lock 0xa33a5be14eae1bb8 00000080:00010000:3.0:1773941321.750545:0:5510:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xa33a5be14eae1bb8 from it ffffa10c41697cb8 00010000:00010000:3.0:1773941321.750549:0:5510:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1773941321.750558:0:5510:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200008:3.0:1773941321.751134:0:5510:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b5b83538908), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1773941321.751139:0:5510:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908),name=/ 00000002:00010000:3.0:1773941321.751160:0:5510:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1773941321.751174:0:5510:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:3.0:1773941321.751183:0:5510:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200000:3.0:1773941321.751189:0:5510:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1773941321.751194:0:5510:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) 00000080:00010000:3.0:1773941321.751197:0:5510:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) for lock 0xa33a5be14eae1bb8 00000080:00010000:3.0:1773941321.751201:0:5510:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xa33a5be14eae1bb8 from it ffffa10c41697cb8 00010000:00010000:3.0:1773941321.751204:0:5510:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:3.0:1773941321.751212:0:5510:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:3.0:1773941321.751304:0:5510:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908),name=/ 00000002:00010000:3.0:1773941321.751312:0:5510:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1773941321.751319:0:5510:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:3.0:1773941321.751328:0:5510:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:3.0:1773941321.751334:0:5510:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1773941321.751338:0:5510:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) 00000080:00010000:3.0:1773941321.751341:0:5510:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) for lock 0xa33a5be14eae1bb8 00000080:00010000:3.0:1773941321.751345:0:5510:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xa33a5be14eae1bb8 from it ffffa10c41697bb0 00010000:00010000:3.0:1773941321.751354:0:5510:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:3.0:1773941321.751362:0:5510:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:3.0:1773941321.751370:0:5510:0:(file.c:6668:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908), inode mode 41ed mask 44 00000080:00200008:3.0:1773941321.751377:0:5510:0:(file.c:1025:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b5b83538908), flags 100000 00000080:00200080:3.0:1773941321.751461:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:3.0:1773941321.751474:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0b91400 uarg=0000000000000000 00800000:00000004:3.0:1773941321.751480:0:5510:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b5b855a9800: age 90, max_age 91 00000100:00100000:3.0:1773941321.751516:0:5510:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277b80 pname:cluuid:pid:xid:nid:opc:job lfs:67bbba49-47e8-4446-a6da-0acb748c20a3:5510:1860112241134720:192.168.204.143@tcp:41: 00000100:00100000:3.0:1773941321.751612:0:5510:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0d8f240 going to sleep for 11 seconds 00000100:00100000:0.0:1773941321.752789:0:5510:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277b80 pname:cluuid:pid:xid:nid:opc:job lfs:67bbba49-47e8-4446-a6da-0acb748c20a3:5510:1860112241134720:192.168.204.143@tcp:41: 00800000:00000004:0.0:1773941321.752806:0:5510:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b5b855a9800: update ffff8b5ba0c3b7c0 cache blocks 317407/353529 objects 1023725/1024000 00000080:00200080:0.0:1773941321.752846:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752856:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752860:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752864:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752867:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752869:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752872:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752873:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752876:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752877:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752880:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752881:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752883:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752886:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752889:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752892:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752894:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752897:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752899:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752901:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752912:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752918:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752923:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752927:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752937:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752942:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752947:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752952:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752956:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752960:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752965:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752969:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752974:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752978:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752982:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752987:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.752991:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.752996:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753001:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753005:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753010:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753014:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753018:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753022:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753027:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753036:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753042:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753046:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753050:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753053:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753057:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753059:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753063:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753066:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753070:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753073:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753076:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753080:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753084:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753087:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753091:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753094:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753097:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753101:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753105:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753109:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753111:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753113:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753116:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753118:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753125:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753128:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753130:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753132:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753134:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753136:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753138:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753139:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753151:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753156:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753159:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753161:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753163:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753165:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753167:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753168:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753170:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753172:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753174:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753176:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753177:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753179:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753181:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753183:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753185:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753192:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753197:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753200:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753204:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753208:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753212:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753215:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753219:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753223:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753226:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753228:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753230:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753232:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753234:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753235:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753237:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753239:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753241:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753242:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753244:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753246:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753248:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753249:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753251:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753253:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753260:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753263:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753265:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753267:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753269:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753271:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753273:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00800000:00000080:0.0:1773941321.753274:0:5510:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.753276:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.753280:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00020000:00000004:0.0:1773941321.753285:0:5510:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b5b855a9800: age -911, max_age 91 00000100:00100000:0.0:1773941321.753302:0:5510:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a9c00 pname:cluuid:pid:xid:nid:opc:job lfs:67bbba49-47e8-4446-a6da-0acb748c20a3:5510:1860112241134848:192.168.204.143@tcp:13: 00000100:00100000:0.0:1773941321.753337:0:5510:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b84cfa480 going to sleep for 11 seconds 00010000:00080000:0.0:1773941321.754110:0:5510:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b5b850a9c00 x1860112241134848/t0(0) o13->lustre-OST0000-osc-ffff8b5b855a9800@192.168.204.143@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941337 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941321.754119:0:5510:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a9c00 pname:cluuid:pid:xid:nid:opc:job lfs:67bbba49-47e8-4446-a6da-0acb748c20a3:5510:1860112241134848:192.168.204.143@tcp:13: 00020000:00000004:0.0:1773941321.754129:0:5510:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b5b855a9800: update ffff8b5ba0c38450 cache blocks 901403/958279 objects 261842/262144 00000080:00200080:0.0:1773941321.754165:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.754171:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00020000:00000004:0.0:1773941321.754174:0:5510:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b5b855a9800: age -911, max_age 91 00000100:00100000:0.0:1773941321.754188:0:5510:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a9c00 pname:cluuid:pid:xid:nid:opc:job lfs:67bbba49-47e8-4446-a6da-0acb748c20a3:5510:1860112241134976:192.168.204.143@tcp:13: 00000100:00100000:0.0:1773941321.754214:0:5510:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b84cfa480 going to sleep for 11 seconds 00010000:00080000:0.0:1773941321.755023:0:5510:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b5b850a9c00 x1860112241134976/t0(0) o13->lustre-OST0001-osc-ffff8b5b855a9800@192.168.204.143@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941337 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941321.755048:0:5510:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a9c00 pname:cluuid:pid:xid:nid:opc:job lfs:67bbba49-47e8-4446-a6da-0acb748c20a3:5510:1860112241134976:192.168.204.143@tcp:13: 00020000:00000004:0.0:1773941321.755057:0:5510:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b5b855a9800: update ffff8b5b98ae5178 cache blocks 901403/958279 objects 261874/262144 00000080:00200080:0.0:1773941321.755079:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755084:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755089:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755091:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755094:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755096:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755100:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755103:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755107:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755111:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755115:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755119:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755124:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755129:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755134:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755139:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755154:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755158:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755162:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755167:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755170:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755180:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755186:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755190:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755196:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755200:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755204:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755207:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755209:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755211:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755213:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755215:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755217:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755219:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755221:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755222:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755224:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755226:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755228:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755229:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755231:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755233:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755235:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755236:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755238:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755240:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755247:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755249:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755251:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755253:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755255:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755257:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755259:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755261:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755263:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755264:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755266:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755268:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755270:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755271:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755273:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755275:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755277:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755278:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755280:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755282:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755284:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755285:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755287:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755289:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755291:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755296:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755299:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755307:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755310:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755312:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755314:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755316:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755318:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755320:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755322:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755324:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755325:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755327:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755329:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755331:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755333:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755334:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755336:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755338:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755340:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755341:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755343:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755345:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755347:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755349:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755355:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755357:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755359:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755361:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755363:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755365:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755367:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755369:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755371:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755372:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755374:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755376:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755378:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755379:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755381:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755383:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755385:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755386:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755388:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755390:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755392:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755394:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755395:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755397:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755399:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755404:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755407:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755409:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200080:0.0:1773941321.755411:0:5510:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffea3eb8340 00020000:00000080:0.0:1773941321.755412:0:5510:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=c00866a4 len=592 karg=ffff8b5ba0bf2000 uarg=0000000000000000 00000080:00200008:0.0:1773941321.755421:0:5510:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b5b83538908), flags 100000 00010000:00010000:3.0:1773941322.399989:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1773941322.595667:0:5699:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: Using TIMEOUT=20 00010000:00010000:3.0:1773941323.423774:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1773941323.707874:0:2346:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.143@tcp (source: 12345-192.168.204.143@tcp) 00000100:00100000:2.0:1773941323.707930:0:2346:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.143@tcp (source: 12345-192.168.204.143@tcp) 00000100:00100000:3.0:1773941323.707955:0:5417:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1860112247444864 00000100:00100000:3.0:1773941323.707968:0:5417:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1860112247444864 00000100:00100000:2.0:1773941323.707997:0:5418:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1860112247444992 00000100:00100000:2.0:1773941323.708008:0:5418:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1860112247444992 00000100:00100000:3.0:1773941323.708027:0:5417:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.143@tcp, seq: 0 00000100:00100000:3.0:1773941323.708042:0:5417:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b5b9021dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:33637e0d-6090-4776-bb84-6d71891db494+4:8740:x1860112247444864:12345-192.168.204.143@tcp:104: 00010000:00010000:3.0:1773941323.708065:0:5417:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1b5d lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0xb574482a8fd85a9d expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941323.708169:0:5417:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1773941323.708175:0:5417:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b5b9021dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:33637e0d-6090-4776-bb84-6d71891db494+4:8740:x1860112247444864:12345-192.168.204.143@tcp:104: Request processed in 138us (313us total) trans 0 rc 0/0 00000100:00100000:3.0:1773941323.708182:0:5417:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.143@tcp, seq: 0 00000100:00100000:3.0:1773941323.708201:0:5417:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.143@tcp, seq: 1 00000100:00100000:3.0:1773941323.708209:0:5417:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b5b9021ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:33637e0d-6090-4776-bb84-6d71891db494+3:8740:x1860112247444992:12345-192.168.204.143@tcp:104: 00010000:00010000:0.0:1773941323.708217:0:5426:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1773941323.708227:0:5426:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1b5d lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0xb574482a8fd85a9d expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941323.708237:0:5417:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1b56 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1400000000000 nid: local remote: 0xb574482a8fd85a96 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941323.708239:0:5426:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138fa00 already unused, calling callback (ffffffffc1482c20) 10000000:00010000:0.0:1773941323.708247:0:5426:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1b5d lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1400400000000 nid: local remote: 0xb574482a8fd85a9d expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941323.708256:0:5426:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1b5d lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1408400000000 nid: local remote: 0xb574482a8fd85a9d expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:0.0:1773941323.708262:0:5426:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1b5d lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1409400000000 nid: local remote: 0xb574482a8fd85a9d expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1773941323.708267:0:5426:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1773941323.708285:0:5417:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1773941323.708293:0:5417:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b5b9021ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:33637e0d-6090-4776-bb84-6d71891db494+3:8740:x1860112247444992:12345-192.168.204.143@tcp:104: Request processed in 85us (365us total) trans 0 rc 0/0 00000100:00100000:3.0:1773941323.708301:0:5417:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.143@tcp, seq: 1 00010000:00010000:0.0:1773941323.708321:0:5426:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1b5d lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb574482a8fd85a9d expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941323.708329:0:5426:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1773941323.708332:0:5425:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1773941323.708357:0:5426:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1b5d lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb574482a8fd85a9d expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941323.708366:0:5426:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1b5d lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb574482a8fd85a9d expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1773941323.708374:0:2352:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33637e0d-6090-4776-bb84-6d71891db494:2352:1860112241135104:192.168.204.143@tcp:103: 00010000:00010000:0.0:1773941323.708410:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941323.708413:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941323.708414:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941323.708591:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941323.708594:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941323.708596:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1773941323.708615:0:5425:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1b56 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0xb574482a8fd85a96 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941323.708624:0:5425:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138ec00 already unused, calling callback (ffffffffc1482c20) 10000000:00010000:1.0:1773941323.708628:0:5425:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1b56 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0xb574482a8fd85a96 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941323.708636:0:5425:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1b56 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0xb574482a8fd85a96 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1773941323.708643:0:5425:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1b56 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0xb574482a8fd85a96 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941323.708649:0:5425:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:1.0:1773941323.708673:0:5425:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1b56 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb574482a8fd85a96 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941323.708692:0:5425:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1773941323.708740:0:5425:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1b56 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb574482a8fd85a96 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941323.708748:0:5425:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1b56 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb574482a8fd85a96 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941323.708766:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1773941323.708768:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1773941323.708770:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1773941323.708779:0:2352:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1773941323.708790:0:2352:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33637e0d-6090-4776-bb84-6d71891db494:2352:1860112241135232:192.168.204.143@tcp:103: 00000100:00100000:1.0:1773941323.709203:0:2352:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33637e0d-6090-4776-bb84-6d71891db494:2352:1860112241135104:192.168.204.143@tcp:103: 00000100:00100000:1.0:1773941323.709522:0:2352:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33637e0d-6090-4776-bb84-6d71891db494:2352:1860112241135232:192.168.204.143@tcp:103: 00010000:00010000:0.0:1773941324.447857:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941325.471811:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1773941325.727917:0:2354:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90274700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33637e0d-6090-4776-bb84-6d71891db494:2354:1860112241135360:192.168.204.143@tcp:400: 00000100:00100000:3.0:1773941325.727938:0:2355:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90274380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:67bbba49-47e8-4446-a6da-0acb748c20a3:2355:1860112241135488:192.168.204.143@tcp:400: 00000100:00100000:2.0:1773941325.728045:0:2354:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:67bbba49-47e8-4446-a6da-0acb748c20a3:2354:1860112241135616:192.168.204.143@tcp:400: 00000100:00100000:3.0:1773941325.728061:0:2355:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:67bbba49-47e8-4446-a6da-0acb748c20a3:2355:1860112241135744:192.168.204.143@tcp:400: 00000100:00100000:3.0:1773941325.729335:0:2355:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90274380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:67bbba49-47e8-4446-a6da-0acb748c20a3:2355:1860112241135488:192.168.204.143@tcp:400: 00010000:00080000:3.0:1773941325.729355:0:2355:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b5b90277480 x1860112241135744/t0(0) o400->lustre-OST0001-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941341 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941325.729360:0:2354:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90274700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33637e0d-6090-4776-bb84-6d71891db494:2354:1860112241135360:192.168.204.143@tcp:400: 00000100:00100000:3.0:1773941325.729375:0:2355:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:67bbba49-47e8-4446-a6da-0acb748c20a3:2355:1860112241135744:192.168.204.143@tcp:400: 00010000:00080000:2.0:1773941325.729381:0:2354:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b5b90276d80 x1860112241135616/t0(0) o400->lustre-OST0000-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941341 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941325.729396:0:2354:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:67bbba49-47e8-4446-a6da-0acb748c20a3:2354:1860112241135616:192.168.204.143@tcp:400: 00010000:00010000:1.0:1773941326.495899:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:2.0:1773941327.263834:0:5428:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:2.0:1773941327.263843:0:5428:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-client-ffff8b5b855a9800 from 39 10000000:01000000:2.0:1773941327.263849:0:5428:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:2.0:1773941327.263915:0:5428:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941327.263927:0:5428:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941327.263937:0:5428:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1773941327.263962:0:5428:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021c000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241135872:192.168.204.143@tcp:101: 00000100:00100000:2.0:1773941327.264090:0:5428:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b98124a80 going to sleep for 11 seconds 00000100:00100000:0.0:1773941327.265452:0:5428:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021c000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241135872:192.168.204.143@tcp:101: 00010000:00010000:0.0:1773941327.265476:0:5428:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb574482a8fd85b4c expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941327.265488:0:5428:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xb574482a8fd85b4c expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1773941327.265548:0:5428:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241136000:192.168.204.143@tcp:501: 00000100:00100000:0.0:1773941327.265582:0:5428:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b88ea7480 going to sleep for 11 seconds 00000100:00100000:0.0:1773941327.266313:0:5428:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241136000:192.168.204.143@tcp:501: 00000100:00100000:0.0:1773941327.266343:0:5428:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241136128:192.168.204.143@tcp:503: 00000100:00100000:0.0:1773941327.266403:0:5428:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b88ea7480 going to sleep for 11 seconds 00000100:00100000:0.0:1773941327.267249:0:5428:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241136128:192.168.204.143@tcp:503: 00000100:00100000:0.0:1773941327.267513:0:5917:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8e00 pname:cluuid:pid:xid:nid:opc:job llog_process_th:33637e0d-6090-4776-bb84-6d71891db494:5917:1860112241136256:192.168.204.143@tcp:502: 00000100:00100000:0.0:1773941327.267544:0:5917:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b88ea7cc0 going to sleep for 11 seconds 00000100:00100000:0.0:1773941327.268276:0:5917:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8e00 pname:cluuid:pid:xid:nid:opc:job llog_process_th:33637e0d-6090-4776-bb84-6d71891db494:5917:1860112241136256:192.168.204.143@tcp:502: 00000020:01000000:0.0:1773941327.268305:0:5917:0:(obd_config.c:1923:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:0.0:1773941327.268315:0:5917:0:(obd_config.c:1506:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1773941327.268317:0:5917:0:(obd_config.c:1594:class_process_config()) marker 27 (0x1) lustre sys.jobid_var 00000020:00000080:0.0:1773941327.268322:0:5917:0:(obd_config.c:1506:class_process_config()) processing cmd: cf00f 00000100:00100000:0.0:1773941327.268341:0:5917:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8e00 pname:cluuid:pid:xid:nid:opc:job llog_process_th:33637e0d-6090-4776-bb84-6d71891db494:5917:1860112241136384:192.168.204.143@tcp:502:llog_process_th.0 00000100:00100000:0.0:1773941327.268378:0:5917:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b88ea7cc0 going to sleep for 11 seconds 00000100:00100000:0.0:1773941327.269068:0:5917:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8e00 pname:cluuid:pid:xid:nid:opc:job llog_process_th:33637e0d-6090-4776-bb84-6d71891db494:5917:1860112241136384:192.168.204.143@tcp:502:llog_process_th.0 00000020:01000000:0.0:1773941327.269082:0:5917:0:(obd_config.c:1923:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1773941327.269085:0:5917:0:(obd_config.c:1506:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1773941327.269087:0:5917:0:(obd_config.c:1594:class_process_config()) marker 27 (0x2) lustre sys.jobid_var 00000040:00080000:0.0:1773941327.269089:0:5917:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 42 count 42 00000020:01000000:2.0:1773941327.269149:0:5428:0:(obd_config.c:2178:class_config_parse_llog()) Processed log lustre-client gen 39-41 (rc=0) 10000000:01000000:2.0:1773941327.269164:0:5428:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:2.0:1773941327.269172:0:5428:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb574482a8fd85b4c expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941327.269180:0:5428:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb574482a8fd85b4c expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1773941327.269196:0:5428:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:2.0:1773941327.269198:0:5428:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fcd28 from 1 10000000:01000000:2.0:1773941327.269200:0:5428:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:2.0:1773941327.269218:0:5428:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150e00/0xa33a5be14eae1bc6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941327.269227:0:5428:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150e00/0xa33a5be14eae1bc6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941327.269235:0:5428:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150e00/0xa33a5be14eae1bc6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1773941327.269251:0:5428:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241136512:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1773941327.269292:0:5428:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b98124300 going to sleep for 11 seconds 00000100:00100000:0.0:1773941327.270055:0:5428:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241136512:192.168.204.143@tcp:101:ll_cfg_requeue.0 00010000:00010000:0.0:1773941327.270064:0:5428:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150e00/0xa33a5be14eae1bc6 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xb574482a8fd85b53 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941327.270072:0:5428:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150e00/0xa33a5be14eae1bc6 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0xb574482a8fd85b53 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1773941327.270096:0:5428:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241136640:192.168.204.143@tcp:501:ll_cfg_requeue.0 00000100:00100000:0.0:1773941327.270120:0:5428:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b88ea7cc0 going to sleep for 11 seconds 00000100:00100000:0.0:1773941327.270753:0:5428:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241136640:192.168.204.143@tcp:501:ll_cfg_requeue.0 10000000:01000000:0.0:1773941327.270768:0:5428:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:0.0:1773941327.270773:0:5428:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150e00/0xa33a5be14eae1bc6 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xb574482a8fd85b53 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941327.270779:0:5428:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150e00/0xa33a5be14eae1bc6 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xb574482a8fd85b53 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941327.519833:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1773941328.131065:0:2346:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.143@tcp (source: 12345-192.168.204.143@tcp) 00000100:00100000:2.0:1773941328.131155:0:5418:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1860112247447040 00000100:00100000:2.0:1773941328.131168:0:5418:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1860112247447040 00000100:00100000:2.0:1773941328.131242:0:5418:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.143@tcp, seq: 2 00000100:00100000:2.0:1773941328.131257:0:5418:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b5b90277480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:33637e0d-6090-4776-bb84-6d71891db494+3:8793:x1860112247447040:12345-192.168.204.143@tcp:104: 00010000:00010000:2.0:1773941328.131281:0:5418:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.204.143@tcp lock: ffff8b5b80031400/0xa33a5be14eae1b8e lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0xb574482a8fd85aab expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941328.131385:0:5418:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1773941328.131394:0:5418:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b5b90277480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:33637e0d-6090-4776-bb84-6d71891db494+3:8793:x1860112247447040:12345-192.168.204.143@tcp:104: Request processed in 141us (341us total) trans 0 rc 0/0 00000100:00100000:2.0:1773941328.131404:0:5418:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.143@tcp, seq: 2 00010000:00010000:0.0:1773941328.131475:0:5889:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1773941328.131484:0:5889:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b80031400/0xa33a5be14eae1b8e lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0xb574482a8fd85aab expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941328.131493:0:5889:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b80031400 already unused, calling callback (ffffffffc1482c20) 10000000:00010000:0.0:1773941328.131497:0:5889:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b80031400/0xa33a5be14eae1b8e lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0xb574482a8fd85aab expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941328.131507:0:5889:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b80031400/0xa33a5be14eae1b8e lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0xb574482a8fd85aab expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:0.0:1773941328.131513:0:5889:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b80031400/0xa33a5be14eae1b8e lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0xb574482a8fd85aab expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1773941328.131518:0:5889:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:0.0:1773941328.131579:0:5889:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.143@tcp lock: ffff8b5b80031400/0xa33a5be14eae1b8e lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb574482a8fd85aab expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941328.131584:0:5889:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1773941328.131617:0:5889:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b80031400/0xa33a5be14eae1b8e lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb574482a8fd85aab expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941328.131623:0:5889:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b80031400/0xa33a5be14eae1b8e lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb574482a8fd85aab expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941328.131647:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941328.131648:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941328.131650:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1773941328.131657:0:2352:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33637e0d-6090-4776-bb84-6d71891db494:2352:1860112241136768:192.168.204.143@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1773941328.132573:0:2352:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33637e0d-6090-4776-bb84-6d71891db494:2352:1860112241136768:192.168.204.143@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1773941328.543926:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:2.0:1773941328.735839:0:5428:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:2.0:1773941328.735848:0:5428:0:(mgc_request.c:1813:mgc_process_log()) Process log params-ffff8b5b855a9800 from 4 10000000:01000000:2.0:1773941328.735879:0:5428:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:2.0:1773941328.735937:0:5428:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941328.735950:0:5428:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941328.735961:0:5428:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1773941328.735990:0:5428:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021fb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241136896:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1773941328.736106:0:5428:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b850d7cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1773941328.737655:0:5428:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021fb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241136896:192.168.204.143@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1773941328.737675:0:5428:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb574482a8fd85b61 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941328.737685:0:5428:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xb574482a8fd85b61 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1773941328.737766:0:5428:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021fb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241137024:192.168.204.143@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1773941328.737811:0:5428:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b850d7cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1773941328.738596:0:5428:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021fb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241137024:192.168.204.143@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1773941328.738640:0:5428:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021fb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241137152:192.168.204.143@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1773941328.738680:0:5428:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b850d7cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1773941328.739582:0:5428:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021fb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:33637e0d-6090-4776-bb84-6d71891db494:5428:1860112241137152:192.168.204.143@tcp:503:ll_cfg_requeue.0 00000100:00100000:0.0:1773941328.739965:0:5980:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8000 pname:cluuid:pid:xid:nid:opc:job llog_process_th:33637e0d-6090-4776-bb84-6d71891db494:5980:1860112241137280:192.168.204.143@tcp:502:llog_process_th.0 00000100:00100000:0.0:1773941328.740033:0:5980:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b88ea7240 going to sleep for 11 seconds 00000100:00100000:0.0:1773941328.740842:0:5980:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8000 pname:cluuid:pid:xid:nid:opc:job llog_process_th:33637e0d-6090-4776-bb84-6d71891db494:5980:1860112241137280:192.168.204.143@tcp:502:llog_process_th.0 00000020:01000000:0.0:1773941328.740875:0:5980:0:(obd_config.c:1923:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:0.0:1773941328.740888:0:5980:0:(obd_config.c:1506:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1773941328.740892:0:5980:0:(obd_config.c:1594:class_process_config()) marker 3 (0x1) general lod.*.mdt_hash 00000020:01000000:0.0:1773941328.740900:0:5980:0:(obd_config.c:2038:class_config_llog_handler()) cmd ce032, instance name: general-ffff8b5b855a9800 00000020:00000080:0.0:1773941328.740904:0:5980:0:(obd_config.c:1506:class_process_config()) processing cmd: ce032 00000100:00100000:0.0:1773941328.741028:0:5980:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8000 pname:cluuid:pid:xid:nid:opc:job llog_process_th:33637e0d-6090-4776-bb84-6d71891db494:5980:1860112241137408:192.168.204.143@tcp:502:llog_process_th.0 00000100:00100000:0.0:1773941328.741080:0:5980:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b88ea7240 going to sleep for 11 seconds 00000100:00100000:0.0:1773941328.742045:0:5980:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8000 pname:cluuid:pid:xid:nid:opc:job llog_process_th:33637e0d-6090-4776-bb84-6d71891db494:5980:1860112241137408:192.168.204.143@tcp:502:llog_process_th.0 00000020:01000000:0.0:1773941328.742068:0:5980:0:(obd_config.c:1923:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1773941328.742073:0:5980:0:(obd_config.c:1506:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1773941328.742075:0:5980:0:(obd_config.c:1594:class_process_config()) marker 3 (0x2) general lod.*.mdt_hash 00000040:00080000:0.0:1773941328.742079:0:5980:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 7 count 7 00000020:01000000:3.0:1773941328.742174:0:5428:0:(obd_config.c:2178:class_config_parse_llog()) Processed log params gen 4-6 (rc=0) 10000000:01000000:3.0:1773941328.742222:0:5428:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:3.0:1773941328.742233:0:5428:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb574482a8fd85b61 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941328.742243:0:5428:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb574482a8fd85b61 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941329.567869:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941330.591915:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1773941330.847926:0:2354:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33637e0d-6090-4776-bb84-6d71891db494:2354:1860112241137536:192.168.204.143@tcp:400:kworker.0 00000100:00100000:2.0:1773941330.847943:0:2355:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:67bbba49-47e8-4446-a6da-0acb748c20a3:2355:1860112241137664:192.168.204.143@tcp:400:kworker.0 00000100:00100000:3.0:1773941330.848028:0:2354:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:67bbba49-47e8-4446-a6da-0acb748c20a3:2354:1860112241137792:192.168.204.143@tcp:400:kworker.0 00000100:00100000:2.0:1773941330.848030:0:2355:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:67bbba49-47e8-4446-a6da-0acb748c20a3:2355:1860112241137920:192.168.204.143@tcp:400:kworker.0 00000100:00100000:2.0:1773941330.849313:0:2354:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33637e0d-6090-4776-bb84-6d71891db494:2354:1860112241137536:192.168.204.143@tcp:400:kworker.0 00000100:00100000:3.0:1773941330.849317:0:2355:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:67bbba49-47e8-4446-a6da-0acb748c20a3:2355:1860112241137664:192.168.204.143@tcp:400:kworker.0 00010000:00080000:3.0:1773941330.849341:0:2355:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b5b9021c000 x1860112241137920/t0(0) o400->lustre-OST0001-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941346 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00080000:2.0:1773941330.849341:0:2354:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b5b9021ca80 x1860112241137792/t0(0) o400->lustre-OST0000-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941346 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941330.849377:0:2355:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:67bbba49-47e8-4446-a6da-0acb748c20a3:2355:1860112241137920:192.168.204.143@tcp:400:kworker.0 00000100:00100000:2.0:1773941330.849382:0:2354:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:67bbba49-47e8-4446-a6da-0acb748c20a3:2354:1860112241137792:192.168.204.143@tcp:400:kworker.0 00010000:00010000:1.0:1773941331.615998:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941332.639885:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941333.663861:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1773941334.689002:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1773941335.097983:0:6262:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg443-client.virtnet: executing check_logdir /tmp/testlogs/ 00010000:00010000:0.0:1773941335.711849:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1773941335.967900:0:2354:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33637e0d-6090-4776-bb84-6d71891db494:2354:1860112241138048:192.168.204.143@tcp:400:kworker.0 00000100:00100000:2.0:1773941335.967904:0:2355:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:67bbba49-47e8-4446-a6da-0acb748c20a3:2355:1860112241138176:192.168.204.143@tcp:400:kworker.0 00000100:00100000:2.0:1773941335.968003:0:2355:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:67bbba49-47e8-4446-a6da-0acb748c20a3:2355:1860112241138432:192.168.204.143@tcp:400:kworker.0 00000100:00100000:3.0:1773941335.968006:0:2354:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:67bbba49-47e8-4446-a6da-0acb748c20a3:2354:1860112241138304:192.168.204.143@tcp:400:kworker.0 00000100:00100000:2.0:1773941335.969295:0:2354:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33637e0d-6090-4776-bb84-6d71891db494:2354:1860112241138048:192.168.204.143@tcp:400:kworker.0 00000100:00100000:2.0:1773941335.969368:0:2355:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:67bbba49-47e8-4446-a6da-0acb748c20a3:2355:1860112241138176:192.168.204.143@tcp:400:kworker.0 00010000:00080000:2.0:1773941335.969389:0:2355:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b5b9021df80 x1860112241138432/t0(0) o400->lustre-OST0001-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941351 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00080000:3.0:1773941335.969404:0:2354:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b5b9021d180 x1860112241138304/t0(0) o400->lustre-OST0000-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941351 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941335.969405:0:2355:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:67bbba49-47e8-4446-a6da-0acb748c20a3:2355:1860112241138432:192.168.204.143@tcp:400:kworker.0 00000100:00100000:3.0:1773941335.969425:0:2354:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:67bbba49-47e8-4446-a6da-0acb748c20a3:2354:1860112241138304:192.168.204.143@tcp:400:kworker.0 00000001:02000400:3.0:1773941336.513446:0:6524:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg443-client.virtnet: executing yml_node 00010000:00010000:3.0:1773941336.736937:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1773941337.759805:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1773941337.894157:0:6963:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: Client: 2.17.50.193 00000001:02000400:2.0:1773941338.711658:0:7117:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: MDS: 2.17.50.193 00010000:00010000:0.0:1773941338.783775:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1773941339.549954:0:7271:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: OSS: 2.17.50.193 00010000:00010000:3.0:1773941339.808511:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1773941340.070357:0:7416:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: -----============= acceptance-small: conf-sanity ============----- Thu Mar 19 13:28:59 EDT 2026 00010000:00010000:1.0:1773941340.831835:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1773941341.088844:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b5b9026e000 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:02000000:3.0:1773941341.088851:0:37:0:(import.c:2088:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffff8b5b855a9800: disconnect after 20s idle 00000100:00100000:2.0:1773941341.089204:0:2354:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90274380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33637e0d-6090-4776-bb84-6d71891db494:2354:1860112241138560:192.168.204.143@tcp:400:kworker.0 00000100:00100000:2.0:1773941341.089288:0:2354:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1773941341.089294:0:2354:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:67bbba49-47e8-4446-a6da-0acb748c20a3:2354:1860112241138688:192.168.204.143@tcp:400:kworker.0 00000100:00100000:0.0:1773941341.092375:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:67bbba49-47e8-4446-a6da-0acb748c20a3:2351:1860112241138816:192.168.204.143@tcp:9:kworker.0 00000100:00080000:2.0:1773941341.092518:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f22000 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:02000000:2.0:1773941341.092524:0:37:0:(import.c:2088:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffff8b5b855a9800: disconnect after 20s idle 00000100:00100000:1.0:1773941341.092580:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:67bbba49-47e8-4446-a6da-0acb748c20a3:2351:1860112241138944:192.168.204.143@tcp:9:kworker.0 00000100:00100000:2.0:1773941341.092609:0:2354:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90274380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33637e0d-6090-4776-bb84-6d71891db494:2354:1860112241138560:192.168.204.143@tcp:400:kworker.0 00000100:00100000:3.0:1773941341.092920:0:2354:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:67bbba49-47e8-4446-a6da-0acb748c20a3:2354:1860112241138688:192.168.204.143@tcp:400:kworker.0 00010000:00080000:1.0:1773941341.093552:0:2351:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b5b90276680 x1860112241138816/t0(0) o9->lustre-OST0000-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941351 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941341.093564:0:2351:0:(import.c:1998:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff8b5b90276680 x1860112241138816/t0(0) o9->lustre-OST0000-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941351 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941341.093571:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b9026e000 lustre-OST0000_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:1.0:1773941341.093575:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:67bbba49-47e8-4446-a6da-0acb748c20a3:2351:1860112241138816:192.168.204.143@tcp:9:kworker.0 00010000:00080000:0.0:1773941341.093821:0:2351:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b5b9021df80 x1860112241138944/t0(0) o9->lustre-OST0001-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941351 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941341.093837:0:2351:0:(import.c:1998:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff8b5b9021df80 x1860112241138944/t0(0) o9->lustre-OST0001-osc-ffff8b5b855a9800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941351 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941341.093853:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f22000 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:0.0:1773941341.093859:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:67bbba49-47e8-4446-a6da-0acb748c20a3:2351:1860112241138944:192.168.204.143@tcp:9:kworker.0 00010000:00010000:0.0:1773941341.855826:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1773941342.879839:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1773941343.903837:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941344.927840:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1773941345.735605:0:7962:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: excepting tests: 32newtarball 00010000:00010000:0.0:1773941345.951787:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1773941346.208903:0:2353:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:33637e0d-6090-4776-bb84-6d71891db494:2353:1860112241139072:192.168.204.143@tcp:400:kworker.0 00000100:00100000:1.0:1773941346.208906:0:2352:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:67bbba49-47e8-4446-a6da-0acb748c20a3:2352:1860112241139200:192.168.204.143@tcp:400:kworker.0 00000100:00100000:1.0:1773941346.210161:0:2353:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:33637e0d-6090-4776-bb84-6d71891db494:2353:1860112241139072:192.168.204.143@tcp:400:kworker.0 00000100:00100000:1.0:1773941346.210213:0:2352:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:67bbba49-47e8-4446-a6da-0acb748c20a3:2352:1860112241139200:192.168.204.143@tcp:400:kworker.0 00000001:02000400:1.0:1773941346.239026:0:8099:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: skipping tests SLOW=no: 45 69 106 111 114 00000080:00200008:0.0:1773941346.605639:0:8209:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b5b83538908), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1773941346.605652:0:8209:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908),name=/ 00000002:00010000:0.0:1773941346.605687:0:8209:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1773941346.605727:0:8209:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:0.0:1773941346.605744:0:8209:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:0.0:1773941346.605753:0:8209:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1773941346.605761:0:8209:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) 00000080:00010000:0.0:1773941346.605765:0:8209:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) for lock 0xa33a5be14eae1bb8 00000080:00010000:0.0:1773941346.605772:0:8209:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xa33a5be14eae1bb8 from it ffffa10c41cbbcb8 00010000:00010000:0.0:1773941346.605776:0:8209:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:0.0:1773941346.605786:0:8209:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200000:2.0:1773941346.804117:0:8210:0:(llite_lib.c:3557:ll_umount_begin()) VFS Op: superblock ffff8b5b855a9800 count 1 active 9 00800000:00000080:2.0:1773941346.804126:0:8210:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b855a9800: cmd=4240666a len=576 karg=ffff8b5b83f09000 uarg=0000000000000000 00000002:00000080:2.0:1773941346.804134:0:8210:0:(mdc_request.c:2300:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b5b855a9800: cmd=4240666a len=576 karg=ffff8b5b83f09000 uarg=0000000000000000 00000100:02000400:2.0:1773941346.804138:0:8210:0:(recover.c:247:ptlrpc_set_import_active()) setting import lustre-MDT0000_UUID INACTIVE by administrator request 00000100:00080000:2.0:1773941346.806002:0:8210:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0000_UUID INVALID 00000100:00100000:2.0:1773941346.806006:0:8210:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b5b855a9800: committing for last_committed 0 gen 2 00000100:00100000:2.0:1773941346.806014:0:8210:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00010000:00010000:2.0:1773941346.806122:0:8210:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10e400000000 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1773941346.806141:0:8210:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1773941346.806150:0:8210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b5b855a9800 lock: ffff8b5bc2150600/0xa33a5be14eae1bb8 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x490f400000000 nid: local remote: 0xb574482a8fd85b3e expref: -99 pid: 5458 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00020000:00000080:2.0:1773941346.807810:0:8210:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b855a9800: cmd=4240666a len=576 karg=ffff8b5b83f09000 uarg=0000000000000000 00000008:00000080:2.0:1773941346.807818:0:8210:0:(osc_request.c:3548:osc_iocontrol()) lustre-OST0000-osc-ffff8b5b855a9800: cmd=4240666a len=576 karg=ffff8b5b83f09000 uarg=0000000000000000 00000100:02000400:2.0:1773941346.807820:0:8210:0:(recover.c:247:ptlrpc_set_import_active()) setting import lustre-OST0000_UUID INACTIVE by administrator request 00020000:01000000:2.0:1773941346.807826:0:8210:0:(lov_obd.c:378:lov_set_osc_active()) deactivate OSC lustre-OST0000_UUID 00000080:00000004:2.0:1773941346.807829:0:8210:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1773941346.807833:0:8210:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1773941346.807836:0:8210:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0000_UUID INVALID 00000100:00100000:2.0:1773941346.807837:0:8210:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b5b855a9800: committing for last_committed 0 gen 2 00020000:01000000:2.0:1773941346.807840:0:8210:0:(lov_obd.c:389:lov_set_osc_active()) Marking OSC lustre-OST0000_UUID inactive 00000080:00000004:2.0:1773941346.807842:0:8210:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1773941346.807843:0:8210:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:2.0:1773941346.807844:0:8210:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000008:00000080:2.0:1773941346.807973:0:8210:0:(osc_request.c:3548:osc_iocontrol()) lustre-OST0001-osc-ffff8b5b855a9800: cmd=4240666a len=576 karg=ffff8b5b83f09000 uarg=0000000000000000 00000100:02000400:2.0:1773941346.807975:0:8210:0:(recover.c:247:ptlrpc_set_import_active()) setting import lustre-OST0001_UUID INACTIVE by administrator request 00020000:01000000:2.0:1773941346.807978:0:8210:0:(lov_obd.c:378:lov_set_osc_active()) deactivate OSC lustre-OST0001_UUID 00000080:00000004:2.0:1773941346.807979:0:8210:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1773941346.807981:0:8210:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1773941346.807983:0:8210:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0001_UUID INVALID 00000100:00100000:2.0:1773941346.807985:0:8210:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b5b855a9800: committing for last_committed 0 gen 2 00020000:01000000:2.0:1773941346.807987:0:8210:0:(lov_obd.c:389:lov_set_osc_active()) Marking OSC lustre-OST0001_UUID inactive 00000080:00000004:2.0:1773941346.807988:0:8210:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1773941346.807989:0:8210:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:2.0:1773941346.807991:0:8210:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00010000:00010000:3.0:1773941346.975892:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941347.999840:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941349.023816:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1773941350.047894:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941351.071826:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1773941351.327870:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 67bbba49-47e8-4446-a6da-0acb748c20a3->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: FULL) 00000100:00100000:2.0:1773941351.327925:0:2354:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33637e0d-6090-4776-bb84-6d71891db494:2354:1860112241139328:192.168.204.143@tcp:400:kworker.0 00000100:00100000:2.0:1773941351.329452:0:2354:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33637e0d-6090-4776-bb84-6d71891db494:2354:1860112241139328:192.168.204.143@tcp:400:kworker.0 00010000:00010000:0.0:1773941352.095876:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1773941353.119919:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941354.143843:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941355.167795:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1773941356.191868:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1773941356.447871:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 67bbba49-47e8-4446-a6da-0acb748c20a3->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: FULL) 00000100:00100000:1.0:1773941356.447910:0:2352:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1773941356.447928:0:2352:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33637e0d-6090-4776-bb84-6d71891db494:2352:1860112241139456:192.168.204.143@tcp:400:kworker.0 00000100:00100000:1.0:1773941356.449093:0:2352:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33637e0d-6090-4776-bb84-6d71891db494:2352:1860112241139456:192.168.204.143@tcp:400:kworker.0 00000080:00200000:2.0:1773941357.093821:0:8210:0:(llite_lib.c:1951:ll_clear_inode()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) 00000080:00200000:2.0:1773941357.093850:0:8210:0:(llite_lib.c:1572:ll_put_super()) VFS Op: cfg_instance lustre-client-ffff8b5b855a9800 (sb ffff8b5b855a9800) 00010000:00010000:2.0:1773941357.093885:0:8210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xb574482a8fd85b4c expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941357.093896:0:8210:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0xb574482a8fd85b4c expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941357.093905:0:8210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000400000000 nid: local remote: 0xb574482a8fd85b4c expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941357.093913:0:8210:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000400000000 nid: local remote: 0xb574482a8fd85b4c expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941357.093931:0:8210:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1773941357.093945:0:8210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b80031600/0xa33a5be14eae1b87 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb574482a8fd85aa4 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941357.093953:0:8210:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b80031600/0xa33a5be14eae1b87 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xb574482a8fd85aa4 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941357.093960:0:8210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b80031600/0xa33a5be14eae1b87 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xb574482a8fd85aa4 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941357.093965:0:5426:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1773941357.093966:0:8210:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b80031600/0xa33a5be14eae1b87 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xb574482a8fd85aa4 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941357.093975:0:5426:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000400000000 nid: local remote: 0xb574482a8fd85b4c expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941357.093984:0:8210:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1773941357.093985:0:5426:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5bc2150200 already unused, calling callback (ffffffffc1482c20) 10000000:00010000:0.0:1773941357.093992:0:5426:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000400000000 nid: local remote: 0xb574482a8fd85b4c expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941357.093993:0:8210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb574482a8fd85b61 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941357.094000:0:5889:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1773941357.094008:0:5889:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b80031600/0xa33a5be14eae1b87 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xb574482a8fd85aa4 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941357.094014:0:8210:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xb574482a8fd85b61 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941357.094015:0:5426:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1008400000000 nid: local remote: 0xb574482a8fd85b4c expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941357.094016:0:5889:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b80031600 already unused, calling callback (ffffffffc1482c20) 10000000:00010000:1.0:1773941357.094022:0:5889:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b80031600/0xa33a5be14eae1b87 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xb574482a8fd85aa4 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941357.094040:0:8210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xb574482a8fd85b61 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:0.0:1773941357.094040:0:5426:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1009400000000 nid: local remote: 0xb574482a8fd85b4c expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941357.094061:0:8210:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xb574482a8fd85b61 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1773941357.094063:0:5426:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:1.0:1773941357.094064:0:5889:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b80031600/0xa33a5be14eae1b87 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1008400000000 nid: local remote: 0xb574482a8fd85aa4 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1773941357.094070:0:5426:0:(mgc_request.c:944:mgc_blocking_ast()) log lustre-client: stopping, won't requeue 10000000:00010000:1.0:1773941357.094081:0:5889:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b80031600/0xa33a5be14eae1b87 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1009400000000 nid: local remote: 0xb574482a8fd85aa4 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941357.094085:0:5889:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:01000000:1.0:1773941357.094087:0:5889:0:(mgc_request.c:944:mgc_blocking_ast()) log lustre-cliir: stopping, won't requeue 10000000:01000000:1.0:1773941357.094088:0:5889:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-cliir 00010000:00010000:2.0:1773941357.094099:0:8210:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 10000000:01000000:2.0:1773941357.094105:0:8210:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-client 00010000:00010000:0.0:1773941357.094105:0:5426:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85b4c expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941357.094109:0:5889:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.143@tcp lock: ffff8b5b80031600/0xa33a5be14eae1b87 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85aa4 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1773941357.094112:0:8210:0:(mgc_request.c:547:config_log_end()) end config log lustre-client (0) 00010000:00010000:3.0:1773941357.094116:0:5425:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1773941357.094125:0:5889:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1773941357.094125:0:5426:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1773941357.094148:0:5426:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85b4c expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941357.094150:0:5889:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b80031600/0xa33a5be14eae1b87 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85aa4 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941357.094159:0:5426:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150200/0xa33a5be14eae1bbf lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85b4c expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941357.094170:0:5889:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b80031600/0xa33a5be14eae1b87 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85aa4 expref: -99 pid: 5414 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941357.094188:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1773941357.094193:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941357.094193:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1773941357.094198:0:5889:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941357.094200:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941357.094205:0:5426:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1773941357.094216:0:2352:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33637e0d-6090-4776-bb84-6d71891db494:2352:1860112241139712:192.168.204.143@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1773941357.094223:0:2353:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:33637e0d-6090-4776-bb84-6d71891db494:2353:1860112241139584:192.168.204.143@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1773941357.094397:0:8214:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1773941357.094400:0:8214:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1773941357.094401:0:8214:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941357.094418:0:5425:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xb574482a8fd85b61 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941357.094427:0:5425:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5bc2151600 already unused, calling callback (ffffffffc1482c20) 10000000:00010000:0.0:1773941357.094430:0:5425:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xb574482a8fd85b61 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941357.094437:0:5425:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1008400000000 nid: local remote: 0xb574482a8fd85b61 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:0.0:1773941357.094443:0:5425:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1009400000000 nid: local remote: 0xb574482a8fd85b61 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1773941357.094448:0:5425:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 10000000:01000000:0.0:1773941357.094451:0:5425:0:(mgc_request.c:944:mgc_blocking_ast()) log params: stopping, won't requeue 10000000:01000000:0.0:1773941357.094453:0:5425:0:(mgc_request.c:130:config_log_put()) dropping config log params 00010000:00010000:0.0:1773941357.094483:0:5425:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85b61 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941357.094490:0:5425:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1773941357.094510:0:5425:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85b61 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00020000:01000000:2.0:1773941357.094514:0:8210:0:(lov_obd.c:589:lov_del_target()) uuid: lustre-OST0000_UUID idx: 0 gen: 1 exp: ffff8b5b90f21800 active: 0 00020000:01000000:2.0:1773941357.094519:0:8210:0:(lov_obd.c:589:lov_del_target()) uuid: lustre-OST0001_UUID idx: 1 gen: 1 exp: ffff8b5b90f20000 active: 0 00020000:01000000:2.0:1773941357.094522:0:8210:0:(lov_obd.c:64:lov_tgts_putref()) destroying 2 lov targets 00000100:00100000:1.0:1773941357.094526:0:2352:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33637e0d-6090-4776-bb84-6d71891db494:2352:1860112241139840:192.168.204.143@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1773941357.094527:0:5425:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151600/0xa33a5be14eae1bcd lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85b61 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00020000:01000000:2.0:1773941357.094531:0:8210:0:(lov_obd.c:611:__lov_del_obd()) Removing tgt lustre-OST0001_UUID : lustre-OST0001-osc-ffff8b5b855a9800 00020000:01000000:2.0:1773941357.094536:0:8210:0:(lov_obd.c:248:lov_disconnect_obd()) lustre-clilov-ffff8b5b855a9800: disconnecting target lustre-OST0001-osc-ffff8b5b855a9800 00000100:00080000:2.0:1773941357.094544:0:8210:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 67bbba49-47e8-4446-a6da-0acb748c20a3->lustre-OST0001_UUID 00010000:00010000:0.0:1773941357.094561:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941357.094564:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941357.094568:0:5425:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1773941357.094641:0:8210:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f22000 lustre-OST0001_UUID: changing import state from IDLE to CLOSED 00000100:00080000:2.0:1773941357.094649:0:8210:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0001_UUID INVALID 00000100:00100000:2.0:1773941357.094652:0:8210:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b5b855a9800: committing for last_committed 0 gen 3 00000100:00100000:2.0:1773941357.094657:0:8210:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:2.0:1773941357.094849:0:8210:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0xa33a5be14eae1bb1 00000020:00000080:2.0:1773941357.094862:0:8210:0:(genops.c:904:class_export_put()) final put ffff8b5b90f20000/LOV_OSC_UUID 00000020:00000080:0.0:1773941357.094899:0:37:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b90f20000/LOV_OSC_UUID for lustre-OST0001-osc-ffff8b5b855a9800 00000020:01000000:2.0:1773941357.094904:0:8210:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-OST0001-osc-ffff8b5b855a9800 (flags='F') 00000020:00000080:2.0:1773941357.094910:0:8210:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000080:2.0:1773941357.094916:0:8210:0:(obd_config.c:991:class_cleanup()) lustre-OST0001-osc-ffff8b5b855a9800: forcing exports to disconnect: 0/2 00000020:00080000:2.0:1773941357.094920:0:8210:0:(genops.c:1577:class_disconnect_exports()) OBD device 5 (ffff8b5b98ae4d28) has no exports 00000008:01000000:2.0:1773941357.094933:0:8210:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-OST0001-osc-ffff8b5b855a9800: client import never connected 00000100:00080000:2.0:1773941357.094936:0:8210:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0001_UUID INVALID 00000100:00100000:2.0:1773941357.094938:0:8210:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b5b855a9800: committing for last_committed 0 gen 4 00000100:00100000:2.0:1773941357.094941:0:8210:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:0.0:1773941357.095109:0:37:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff8b5b90f22000 for lustre-OST0001-osc-ffff8b5b855a9800 00000020:00000080:2.0:1773941357.095299:0:8210:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000080:2.0:1773941357.095305:0:8210:0:(obd_config.c:919:class_detach()) detach on obd lustre-OST0001-osc-ffff8b5b855a9800 (uuid 67bbba49-47e8-4446-a6da-0acb748c20a3) 00000020:00000080:2.0:1773941357.095307:0:8210:0:(genops.c:904:class_export_put()) final put ffff8b5b90f27000/67bbba49-47e8-4446-a6da-0acb748c20a3 00000020:00000080:2.0:1773941357.095310:0:8210:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b90f27000/67bbba49-47e8-4446-a6da-0acb748c20a3 for lustre-OST0001-osc-ffff8b5b855a9800 00000020:01000000:2.0:1773941357.095314:0:8210:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-OST0001-osc-ffff8b5b855a9800 (67bbba49-47e8-4446-a6da-0acb748c20a3) 00020000:01000000:2.0:1773941357.095511:0:8210:0:(lov_obd.c:611:__lov_del_obd()) Removing tgt lustre-OST0000_UUID : lustre-OST0000-osc-ffff8b5b855a9800 00020000:01000000:2.0:1773941357.095513:0:8210:0:(lov_obd.c:248:lov_disconnect_obd()) lustre-clilov-ffff8b5b855a9800: disconnecting target lustre-OST0000-osc-ffff8b5b855a9800 00000100:00080000:2.0:1773941357.095517:0:8210:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 67bbba49-47e8-4446-a6da-0acb748c20a3->lustre-OST0000_UUID 00000100:00100000:1.0:1773941357.095575:0:2352:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850abb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33637e0d-6090-4776-bb84-6d71891db494:2352:1860112241139840:192.168.204.143@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1773941357.095593:0:2352:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33637e0d-6090-4776-bb84-6d71891db494:2352:1860112241139712:192.168.204.143@tcp:103:ldlm_bl.0 00000100:00080000:2.0:1773941357.095619:0:8210:0:(import.c:65:import_set_state_nolock()) ffff8b5b9026e000 lustre-OST0000_UUID: changing import state from IDLE to CLOSED 00000100:00100000:0.0:1773941357.095621:0:2353:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:33637e0d-6090-4776-bb84-6d71891db494:2353:1860112241139584:192.168.204.143@tcp:103:ldlm_bl.0 00000100:00080000:2.0:1773941357.095625:0:8210:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0000_UUID INVALID 00000100:00100000:2.0:1773941357.095629:0:8210:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b5b855a9800: committing for last_committed 0 gen 3 00000100:00100000:2.0:1773941357.095638:0:8210:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:2.0:1773941357.095808:0:8210:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0xa33a5be14eae1baa 00000020:00000080:2.0:1773941357.095819:0:8210:0:(genops.c:904:class_export_put()) final put ffff8b5b90f21800/LOV_OSC_UUID 00000020:00000080:0.0:1773941357.095840:0:37:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b90f21800/LOV_OSC_UUID for lustre-OST0000-osc-ffff8b5b855a9800 00000020:01000000:2.0:1773941357.095853:0:8210:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-OST0000-osc-ffff8b5b855a9800 (flags='F') 00000020:00000080:2.0:1773941357.095859:0:8210:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000080:2.0:1773941357.095862:0:8210:0:(obd_config.c:991:class_cleanup()) lustre-OST0000-osc-ffff8b5b855a9800: forcing exports to disconnect: 0/2 00000020:00080000:2.0:1773941357.095865:0:8210:0:(genops.c:1577:class_disconnect_exports()) OBD device 4 (ffff8b5ba0c38000) has no exports 00000008:01000000:2.0:1773941357.095875:0:8210:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-OST0000-osc-ffff8b5b855a9800: client import never connected 00000100:00080000:2.0:1773941357.095877:0:8210:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0000_UUID INVALID 00000100:00100000:2.0:1773941357.095879:0:8210:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b5b855a9800: committing for last_committed 0 gen 4 00000100:00100000:2.0:1773941357.095882:0:8210:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:0.0:1773941357.096039:0:37:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff8b5b9026e000 for lustre-OST0000-osc-ffff8b5b855a9800 00000020:00000080:2.0:1773941357.096200:0:8210:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000080:2.0:1773941357.096204:0:8210:0:(obd_config.c:919:class_detach()) detach on obd lustre-OST0000-osc-ffff8b5b855a9800 (uuid 67bbba49-47e8-4446-a6da-0acb748c20a3) 00000020:00000080:2.0:1773941357.096206:0:8210:0:(genops.c:904:class_export_put()) final put ffff8b5b9026d000/67bbba49-47e8-4446-a6da-0acb748c20a3 00000020:00000080:2.0:1773941357.096209:0:8210:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b9026d000/67bbba49-47e8-4446-a6da-0acb748c20a3 for lustre-OST0000-osc-ffff8b5b855a9800 00000020:01000000:2.0:1773941357.096212:0:8210:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-OST0000-osc-ffff8b5b855a9800 (67bbba49-47e8-4446-a6da-0acb748c20a3) 00000020:00000080:2.0:1773941357.096433:0:8210:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0xa33a5be14eae1ba3 00000020:00000080:2.0:1773941357.096437:0:8210:0:(genops.c:904:class_export_put()) final put ffff8b5b90f26000/67bbba49-47e8-4446-a6da-0acb748c20a3 00000020:00000080:0.0:1773941357.096467:0:37:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b90f26000/67bbba49-47e8-4446-a6da-0acb748c20a3 for lustre-clilov-ffff8b5b855a9800 00000100:00080000:2.0:1773941357.096901:0:8210:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 67bbba49-47e8-4446-a6da-0acb748c20a3->lustre-MDT0000_UUID 00000100:00080000:2.0:1773941357.098076:0:8210:0:(import.c:65:import_set_state_nolock()) ffff8b5ba0b00000 lustre-MDT0000_UUID: changing import state from FULL to CLOSED 00000100:00080000:2.0:1773941357.098082:0:8210:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0000_UUID INVALID 00000100:00100000:2.0:1773941357.098085:0:8210:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b5b855a9800: committing for last_committed 0 gen 3 00000100:00100000:2.0:1773941357.098088:0:8210:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:2.0:1773941357.099931:0:8210:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0xa33a5be14eae1b9c 00000020:00000080:2.0:1773941357.099938:0:8210:0:(genops.c:904:class_export_put()) final put ffff8b5b885e0000/67bbba49-47e8-4446-a6da-0acb748c20a3 00000020:00000080:2.0:1773941357.099961:0:8210:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0xa33a5be14eae1b95 00000020:00000080:2.0:1773941357.099965:0:8210:0:(genops.c:904:class_export_put()) final put ffff8b5b885e3800/67bbba49-47e8-4446-a6da-0acb748c20a3 00000020:00000080:0.0:1773941357.099971:0:37:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b885e0000/67bbba49-47e8-4446-a6da-0acb748c20a3 for lustre-MDT0000-mdc-ffff8b5b855a9800 00000020:01000000:2.0:1773941357.099982:0:8210:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-clilov-ffff8b5b855a9800 (flags='F') 00000020:00000080:0.0:1773941357.099982:0:37:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b885e3800/67bbba49-47e8-4446-a6da-0acb748c20a3 for lustre-clilmv-ffff8b5b855a9800 00000020:00000080:2.0:1773941357.099990:0:8210:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000080:2.0:1773941357.100001:0:8210:0:(obd_config.c:991:class_cleanup()) lustre-clilov-ffff8b5b855a9800: forcing exports to disconnect: 0/1 00000020:00080000:2.0:1773941357.100005:0:8210:0:(genops.c:1577:class_disconnect_exports()) OBD device 1 (ffff8b5ba0c399b8) has no exports 00000020:00000080:2.0:1773941357.100028:0:8210:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000080:2.0:1773941357.100032:0:8210:0:(obd_config.c:919:class_detach()) detach on obd lustre-clilov-ffff8b5b855a9800 (uuid 67bbba49-47e8-4446-a6da-0acb748c20a3) 00000020:00000080:2.0:1773941357.100035:0:8210:0:(genops.c:904:class_export_put()) final put ffff8b5ba0b04000/67bbba49-47e8-4446-a6da-0acb748c20a3 00000020:00000080:2.0:1773941357.100037:0:8210:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5ba0b04000/67bbba49-47e8-4446-a6da-0acb748c20a3 for lustre-clilov-ffff8b5b855a9800 00000020:01000000:2.0:1773941357.100042:0:8210:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-clilov-ffff8b5b855a9800 (67bbba49-47e8-4446-a6da-0acb748c20a3) 00000020:01000000:2.0:1773941357.100050:0:8210:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-clilmv-ffff8b5b855a9800 (flags='F') 00000020:00000080:2.0:1773941357.100053:0:8210:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000080:2.0:1773941357.100055:0:8210:0:(obd_config.c:991:class_cleanup()) lustre-clilmv-ffff8b5b855a9800: forcing exports to disconnect: 0/1 00000020:00080000:2.0:1773941357.100057:0:8210:0:(genops.c:1577:class_disconnect_exports()) OBD device 2 (ffff8b5ba0c3cd28) has no exports 00000020:00000080:2.0:1773941357.100094:0:8210:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000080:2.0:1773941357.100098:0:8210:0:(obd_config.c:919:class_detach()) detach on obd lustre-clilmv-ffff8b5b855a9800 (uuid 67bbba49-47e8-4446-a6da-0acb748c20a3) 00000020:00000080:2.0:1773941357.100100:0:8210:0:(genops.c:904:class_export_put()) final put ffff8b5ba0b01800/67bbba49-47e8-4446-a6da-0acb748c20a3 00000020:00000080:2.0:1773941357.100103:0:8210:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5ba0b01800/67bbba49-47e8-4446-a6da-0acb748c20a3 for lustre-clilmv-ffff8b5b855a9800 00000020:01000000:2.0:1773941357.100106:0:8210:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-clilmv-ffff8b5b855a9800 (67bbba49-47e8-4446-a6da-0acb748c20a3) 00000020:01000000:2.0:1773941357.100225:0:8210:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-MDT0000-mdc-ffff8b5b855a9800 (flags='F') 00000020:00000080:2.0:1773941357.100228:0:8210:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000080:2.0:1773941357.100231:0:8210:0:(obd_config.c:991:class_cleanup()) lustre-MDT0000-mdc-ffff8b5b855a9800: forcing exports to disconnect: 0/2 00000020:00080000:2.0:1773941357.100233:0:8210:0:(genops.c:1577:class_disconnect_exports()) OBD device 3 (ffff8b5ba0c3b370) has no exports 00000008:01000000:2.0:1773941357.100243:0:8210:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-MDT0000-mdc-ffff8b5b855a9800: client import never connected 00000100:00080000:2.0:1773941357.100246:0:8210:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0000_UUID INVALID 00000100:00100000:2.0:1773941357.100248:0:8210:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b5b855a9800: committing for last_committed 0 gen 4 00000100:00100000:2.0:1773941357.100252:0:8210:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:1.0:1773941357.102014:0:8:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff8b5ba0b00000 for lustre-MDT0000-mdc-ffff8b5b855a9800 00000020:00000080:2.0:1773941357.102228:0:8210:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000080:2.0:1773941357.102235:0:8210:0:(obd_config.c:919:class_detach()) detach on obd lustre-MDT0000-mdc-ffff8b5b855a9800 (uuid 67bbba49-47e8-4446-a6da-0acb748c20a3) 00000020:00000080:2.0:1773941357.102238:0:8210:0:(genops.c:904:class_export_put()) final put ffff8b5ba0b03800/67bbba49-47e8-4446-a6da-0acb748c20a3 00000020:00000080:2.0:1773941357.102241:0:8210:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5ba0b03800/67bbba49-47e8-4446-a6da-0acb748c20a3 for lustre-MDT0000-mdc-ffff8b5b855a9800 00000020:01000000:2.0:1773941357.102247:0:8210:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-MDT0000-mdc-ffff8b5b855a9800 (67bbba49-47e8-4446-a6da-0acb748c20a3) 00000080:02000400:3.0:1773941357.103842:0:8210:0:(llite_lib.c:1617:ll_put_super()) Unmounted lustre-client 00000020:01000000:3.0:1773941357.105327:0:8210:0:(obd_config.c:1233:class_del_profile()) Del profile lustre-client 00000020:01000004:3.0:1773941357.105425:0:8210:0:(obd_mount.c:911:lustre_common_put_super()) dropping sb ffff8b5b855a9800 00000100:00080000:3.0:1773941357.105436:0:8210:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 33637e0d-6090-4776-bb84-6d71891db494->MGS 00010000:00010000:3.0:1773941357.105455:0:8210:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150e00/0xa33a5be14eae1bc6 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1008400000000 nid: local remote: 0xb574482a8fd85b53 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1773941357.105493:0:8210:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150e00/0xa33a5be14eae1bc6 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1009400000000 nid: local remote: 0xb574482a8fd85b53 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1773941357.105502:0:8210:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:01000000:3.0:1773941357.105506:0:8210:0:(mgc_request.c:954:mgc_blocking_ast()) log lustre-s: disconnecting, won't requeue 10000000:01000000:3.0:1773941357.105510:0:8210:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-sptlrpc 00010000:00010000:3.0:1773941357.105569:0:8210:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150e00/0xa33a5be14eae1bc6 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85b53 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941357.105579:0:8210:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:3.0:1773941357.105599:0:8210:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b890dfe40 going to sleep for 0 seconds 00000100:00100000:3.0:1773941357.105613:0:8210:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276a00 pname:cluuid:pid:xid:nid:opc:job umount:33637e0d-6090-4776-bb84-6d71891db494:8210:1860112241139968:192.168.204.143@tcp:103:umount.0 00010000:00010000:3.0:1773941357.105628:0:8210:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:3.0:1773941357.105641:0:8210:0:(ldlm_request.c:2277:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:3.0:1773941357.105645:0:8210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150e00/0xa33a5be14eae1bc6 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85b53 expref: -99 pid: 5428 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:3.0:1773941357.105698:0:8210:0:(import.c:65:import_set_state_nolock()) ffff8b5ba0b0f000 MGS: changing import state from FULL to CONNECTING 00000100:00100000:3.0:1773941357.105800:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33637e0d-6090-4776-bb84-6d71891db494:2351:1860112241140096:192.168.204.143@tcp:251:umount.0 00000100:00080000:0.0:1773941357.106962:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5ba0b0f000 MGS: changing import state from CONNECTING to CLOSED 10000000:01000000:0.0:1773941357.106967:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941357.106969:0:2351:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:0.0:1773941357.106971:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 00000100:00100000:0.0:1773941357.106983:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33637e0d-6090-4776-bb84-6d71891db494:2351:1860112241140096:192.168.204.143@tcp:251:umount.0 00000100:00080000:3.0:1773941357.106995:0:8210:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 10000000:01000000:3.0:1773941357.107013:0:8210:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 00000100:00100000:3.0:1773941357.107017:0:8210:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:3.0:1773941357.107020:0:8210:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000080:3.0:1773941357.107032:0:8210:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0xa33a5be14eae1b4f 00000020:00000080:3.0:1773941357.107043:0:8210:0:(genops.c:904:class_export_put()) final put ffff8b5b90db6000/33637e0d-6090-4776-bb84-6d71891db494 00000020:01000000:3.0:1773941357.107065:0:8210:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of MGC192.168.204.143@tcp (flags='') 00000020:00000080:3.0:1773941357.107071:0:8210:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000080:3.0:1773941357.107079:0:8210:0:(obd_config.c:991:class_cleanup()) MGC192.168.204.143@tcp: forcing exports to disconnect: 0/3 00000020:00080000:3.0:1773941357.107085:0:8210:0:(genops.c:1577:class_disconnect_exports()) OBD device 0 (ffff8b5b982fcd28) has no exports 00000020:00000080:1.0:1773941357.107091:0:8:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b90db6000/33637e0d-6090-4776-bb84-6d71891db494 for MGC192.168.204.143@tcp 10000000:01000000:2.0:1773941357.107142:0:5428:0:(mgc_request.c:708:mgc_requeue_thread()) Ending requeue thread 10000000:01000000:3.0:1773941357.107155:0:8210:0:(obd_class.h:663:obd_cleanup_client_import()) MGC192.168.204.143@tcp: client import never connected 00000100:00080000:3.0:1773941357.107159:0:8210:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 10000000:01000000:3.0:1773941357.107162:0:8210:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 00000100:00100000:3.0:1773941357.107166:0:8210:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:3.0:1773941357.107168:0:8210:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000080:1.0:1773941357.107227:0:8:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff8b5ba0b0f000 for MGC192.168.204.143@tcp 00000020:00000080:3.0:1773941357.107278:0:8210:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000080:3.0:1773941357.107286:0:8210:0:(obd_config.c:919:class_detach()) detach on obd MGC192.168.204.143@tcp (uuid 33637e0d-6090-4776-bb84-6d71891db494) 00000020:00000080:3.0:1773941357.107290:0:8210:0:(genops.c:904:class_export_put()) final put ffff8b5b855ab800/33637e0d-6090-4776-bb84-6d71891db494 00000020:00000080:3.0:1773941357.107293:0:8210:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b855ab800/33637e0d-6090-4776-bb84-6d71891db494 for MGC192.168.204.143@tcp 00000020:01000000:3.0:1773941357.107303:0:8210:0:(genops.c:440:class_free_dev()) finishing cleanup of obd MGC192.168.204.143@tcp (33637e0d-6090-4776-bb84-6d71891db494) 00010000:00010000:1.0:1773941357.107601:0:5889:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-1 blwis in pool) 00010000:00010000:0.0:1773941357.107697:0:5426:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-2 blwis in pool) 00010000:00010000:2.0:1773941357.107799:0:8214:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-3 blwis in pool) 00010000:00010000:1.0:1773941357.107860:0:5425:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-4 blwis in pool) 00000100:00100000:1.0:1773941357.108061:0:5416:0:(service.c:3162:ptlrpc_main()) ldlm_cb00_001: service thread [ffff8b5b90649800:5416] 1 exiting: rc = 0 00000100:00100000:0.0:1773941357.108082:0:5415:0:(service.c:3162:ptlrpc_main()) ldlm_cb00_000: service thread [ffff8b5ba216be00:5415] 0 exiting: rc = 1 00000100:00100000:3.0:1773941357.108206:0:5417:0:(service.c:3162:ptlrpc_main()) ldlm_cb01_000: service thread [ffff8b5b90648c00:5417] 0 exiting: rc = 1 00000100:00100000:2.0:1773941357.108264:0:5418:0:(service.c:3162:ptlrpc_main()) ldlm_cb01_001: service thread [ffff8b5b91df4800:5418] 1 exiting: rc = 0 00000100:00100000:3.0:1773941357.108865:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.108892:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941357.109183:0:5419:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_000: service thread [ffff8b5b85ec8a00:5419] 0 exiting: rc = 1 00000100:00100000:0.0:1773941357.109187:0:5420:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_001: service thread [ffff8b5b91df5800:5420] 1 exiting: rc = 0 00000100:00100000:1.0:1773941357.109259:0:5421:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_002: service thread [ffff8b5b85ec8800:5421] 2 exiting: rc = 0 00000100:00100000:3.0:1773941357.109387:0:5424:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_002: service thread [ffff8b5b91df5400:5424] 2 exiting: rc = 0 00000100:00100000:2.0:1773941357.109432:0:5422:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_000: service thread [ffff8b5b91df4a00:5422] 0 exiting: rc = 1 00000100:00100000:2.0:1773941357.109672:0:5423:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_001: service thread [ffff8b5ba030f600:5423] 1 exiting: rc = 0 00000100:00100000:3.0:1773941357.109799:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.109819:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.109837:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.109851:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.109863:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.109873:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.109885:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.109898:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.109911:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.109925:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.109944:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.109956:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.109971:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.109986:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.109998:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110013:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110029:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110043:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110056:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110071:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110087:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110098:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110105:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110118:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110124:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110129:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110136:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110141:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110147:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110153:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110159:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110165:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110170:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110177:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110183:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110189:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110194:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110200:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110206:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110212:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110217:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110223:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110229:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110234:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110240:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110251:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110257:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110263:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110269:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110275:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110283:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110289:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110295:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110301:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110307:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110313:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110319:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110325:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110331:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110336:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110342:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110347:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110353:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110358:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110366:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110373:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110378:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110385:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110391:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110397:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110403:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110410:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110416:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110422:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110428:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110434:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110439:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110445:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110450:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110455:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110467:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110486:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110492:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110498:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110504:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110510:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110515:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110521:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110527:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110534:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110540:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110546:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110551:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110557:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110562:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110568:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110574:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110580:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110586:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110592:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110598:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110603:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110609:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110615:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110621:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110626:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110632:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110638:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110643:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110649:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110655:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110661:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110667:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110674:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110680:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110693:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110700:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110732:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110746:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110753:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110762:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110770:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110776:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110783:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110792:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110802:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110810:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:3.0:1773941357.110815:0:8210:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000020:01000004:3.0:1773941357.111489:0:8210:0:(obd_mount.c:724:lustre_put_lsi()) put ffff8b5b855a9800 1 00000020:01000004:3.0:1773941357.111494:0:8210:0:(obd_mount.c:682:lustre_free_lsi()) Freeing lsi ffff8b5b855ac800 00000001:02000400:1.0:1773941384.857805:0:10177:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg443-client.virtnet: executing wait_import_state_mount FULL mdc.lustre-MDT0000-mdc-*.mds_server_uuid 00000800:00000010:0.0:1773941389.727868:0:2343:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at ffff8b5b99101d00. 00000800:00000200:0.0:1773941389.727892:0:2343:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941389.727898:0:2343:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941389.727905:0:2343:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101d00 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:2.0:1773941389.728026:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000001:02000400:0.0:1773941392.179636:0:10618:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg443-client.virtnet: executing wait_import_state_mount (FULL|IDLE) osc.lustre-OST0000-osc-[-0-9a-f]*.ost_server_uuid 00000800:00000200:0.0:1773941403.039805:0:2343:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941403.039837:0:2343:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941403.039843:0:2343:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101d00 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:3.0:1773941403.039975:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000001:02000400:3.0:1773941403.240379:0:11151:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == conf-sanity test 161b: test mount with multiple IP for a single hostname ========================================================== 13:30:02 (1773941402) 00000001:02000400:2.0:1773941408.726984:0:11556:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg443-client.virtnet: executing wait_import_state_mount FULL mdc.lustre-MDT0000-mdc-*.mds_server_uuid 00000001:02000400:3.0:1773941414.162205:0:11989:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg443-client.virtnet: executing wait_import_state_mount (FULL|IDLE) osc.lustre-OST0000-osc-[-0-9a-f]*.ost_server_uuid 00000080:00000010:1.0:1773941414.258380:0:12017:0:(super25.c:253:lustre_init_fs_context()) kmalloced '(lmd)': 128 at ffff8b5b90edb700. 00000020:00000010:1.0:1773941414.258404:0:12017:0:(obd_mount.c:1388:lustre_parse_monolithic()) kmalloced '(lmd->lmd_params)': 4096 at ffff8b5b90ee2000. 00000020:00000010:1.0:1773941414.258416:0:12017:0:(obd_mount.c:1590:lustre_parse_monolithic()) kmalloced '(lmd->lmd_dev)': 28 at ffff8b5b8441e000. 00000020:00000010:1.0:1773941414.258420:0:12017:0:(obd_mount.c:1646:lustre_parse_monolithic()) kmalloced '(lmd->lmd_profile)': 14 at ffff8b5b84f3f430. 00000020:00000010:1.0:1773941414.258425:0:12017:0:(obd_mount.c:1674:lustre_parse_monolithic()) kmalloced '(lmd->lmd_opts)': 17 at ffff8b5b8441e120. 00000020:00000010:1.0:1773941414.258553:0:12017:0:(obd_mount.c:656:lustre_init_lsi()) kmalloced '(lsi)': 1880 at ffff8b5b885e4800. 00000020:00000010:1.0:1773941414.258569:0:12017:0:(obd_mount.c:345:lustre_start_mgc()) kmalloced '(mgcname)': 23 at ffff8b5b88570e20. 00000020:00000010:1.0:1773941414.258571:0:12017:0:(obd_mount.c:353:lustre_start_mgc()) kmalloced '(data)': 192 at ffff8b5b85339e40. 00000020:00000010:1.0:1773941414.258578:0:12017:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 88 at ffff8b5b84cfbc60. 00000100:00000010:1.0:1773941414.258583:0:12017:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at ffff8b5b8800ec00. 00000400:00000200:1.0:1773941414.258587:0:12017:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:1.0:1773941414.258591:0:12017:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.143@tcp 00000400:00000200:1.0:1773941414.258597:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.258602:0:12017:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.143@tcp NID flags 0x100001: -17 00000400:00000200:1.0:1773941414.258607:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.258613:0:12017:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1773941414.258615:0:12017:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp: 0. pending discovery 00000400:00000200:1.0:1773941414.258618:0:12017:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.143@tcp 00000100:00000010:1.0:1773941414.258620:0:12017:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at ffff8b5b8800ec00. 00000020:00000010:1.0:1773941414.258623:0:12017:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 88 at ffff8b5b84cfbc60. 00000020:00000010:1.0:1773941414.258628:0:12017:0:(obd_mount.c:487:lustre_start_mgc()) kmalloced '(uuid)': 40 at ffff8b5b82c59440. 00000020:00000010:1.0:1773941414.258639:0:12017:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 120 at ffff8b5b90edb900. 00000020:00000010:1.0:1773941414.258656:0:12017:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at ffff8b5b85514d28. 00000020:00000010:1.0:1773941414.258662:0:12017:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941414.258753:0:12017:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 120 at ffff8b5b90edb900. 00000020:00000010:1.0:1773941414.258757:0:12017:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 104 at ffff8b5b90edb900. 00000001:00000010:1.0:1773941414.258775:0:12017:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at ffff8b5ba0c22900. 00000001:00000010:1.0:1773941414.258780:0:12017:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff8b5b88570c80. 00000001:00000010:1.0:1773941414.258783:0:12017:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5b902a1e00. 00000001:00000010:1.0:1773941414.258784:0:12017:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5b902a1800. 00000001:00000010:1.0:1773941414.258786:0:12017:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5b902a1200. 00000001:00000010:1.0:1773941414.258788:0:12017:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5b902a0c00. 00000020:00000010:1.0:1773941414.258792:0:12017:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b902a0800. 00000020:00000010:1.0:1773941414.258795:0:12017:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8b5b90edb680. 00000020:00000010:1.0:1773941414.258804:0:12017:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8b5ba0c2f4b0. 00008000:00000010:1.0:1773941414.258808:0:12017:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5ba0b143e8. 00000020:00000010:1.0:1773941414.258812:0:12017:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b902a1a00. 00000020:00000010:1.0:1773941414.258815:0:12017:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b903b6200. 00000040:00000010:1.0:1773941414.258818:0:12017:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b903b7200. 00000020:00000010:1.0:1773941414.258821:0:12017:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e6800. 00000020:00000010:1.0:1773941414.258824:0:12017:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b84e8e400. 00040000:00000010:1.0:1773941414.258827:0:12017:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5ba0c22a00. 00040000:00000010:1.0:1773941414.258830:0:12017:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b839c6e00. 10000000:00000010:1.0:1773941414.258832:0:12017:0:(mgc_request.c:852:mgc_device_alloc()) kmalloced '(lu)': 64 at ffff8b5b82c596c0. 00010000:00000010:1.0:1773941414.258840:0:12017:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at ffff8b5b90ee4000. 00010000:00000010:1.0:1773941414.258844:0:12017:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at ffff8b5b90ee5000. 00010000:00000010:1.0:1773941414.258848:0:12017:0:(ldlm_lockd.c:3356:ldlm_setup()) kmalloced '(ldlm_state)': 32 at ffff8b5b88570ca0. 00000100:00000010:1.0:1773941414.258909:0:12017:0:(service.c:745:ptlrpc_register_service()) kmalloced '(service)': 360 at ffff8b5b88c94c00. 00000100:00000010:1.0:1773941414.258915:0:12017:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff8b5b8800d400. 00000100:00000010:1.0:1773941414.258927:0:12017:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff8b5ba0c7c000. 00000100:00000010:1.0:1773941414.258935:0:12017:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff8b5b90ee7000. 00000100:00000010:1.0:1773941414.258939:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8800c000. 00000100:00000010:1.0:1773941414.258943:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d86000. 00000100:00000010:1.0:1773941414.258949:0:12017:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff8b5b8800f800. 00000100:00000010:1.0:1773941414.258956:0:12017:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff8b5b856d0000. 00000100:00000010:1.0:1773941414.258960:0:12017:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff8b5b90ee3000. 00000100:00000010:1.0:1773941414.258963:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8800d800. 00000100:00000010:1.0:1773941414.258966:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d80000. 00000400:00000200:1.0:1773941414.258969:0:12017:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 15 lazy 00000020:00000010:1.0:1773941414.258993:0:12017:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5b8800e000. 00000020:00000010:1.0:1773941414.258996:0:12017:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at ffff8b5b90ee6000. 00000100:00000010:1.0:1773941414.259022:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b85339f00. 00000100:00000010:1.0:1773941414.259025:0:12017:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff8b5b82c59940. 00000100:00000010:1.0:1773941414.259028:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b85339840. 00000100:00000010:1.0:1773941414.259031:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b85339180. 00000100:00000010:1.0:1773941414.259033:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5ba0b97f00. 00000100:00000010:1.0:1773941414.259036:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5ba0b97900. 00000100:00000010:1.0:1773941414.259037:0:12017:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff8b5b82c597c0. 00000100:00000010:1.0:1773941414.259038:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5ba0b97600. 00000100:00000010:1.0:1773941414.259040:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5ba0b6ee40. 00000100:00000010:1.0:1773941414.259042:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5ba0b6e6c0. 00000100:00000200:1.0:1773941414.259060:0:12017:0:(service.c:848:ptlrpc_register_service()) ldlm_cbd: Started, listening on portal 15 00000100:00000010:1.0:1773941414.259062:0:12017:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b88c94600. 00000100:00000010:1.0:1773941414.259223:0:12018:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5b82c59d40. 00000020:00000010:1.0:1773941414.259226:0:12018:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b82c59880. 00000020:00000010:1.0:1773941414.259230:0:12018:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b88c95800. 00000020:00000010:1.0:1773941414.259232:0:12018:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b862a5c00. 00000040:00000010:1.0:1773941414.259233:0:12018:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b862a5600. 00000020:00000010:1.0:1773941414.259235:0:12018:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e5000. 00000020:00000010:1.0:1773941414.259238:0:12018:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b98a05f98. 00000020:00000010:1.0:1773941414.259240:0:12018:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b862a4800. 80000000:00000010:1.0:1773941414.259242:0:12018:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b84cfbf00. 40000000:00000010:1.0:1773941414.259245:0:12018:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b82c595c0. 00100000:00000010:1.0:1773941414.259248:0:12018:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b885e3800. 00040000:00000010:1.0:1773941414.259250:0:12018:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5ba0c22b00. 00040000:00000010:1.0:1773941414.259258:0:12018:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b885e0000. 00040000:00000010:1.0:1773941414.259260:0:12018:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b862a5000. 00000004:00000010:1.0:1773941414.259262:0:12018:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b885e3000. 00000004:00000010:1.0:1773941414.259266:0:12018:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b90ee1000. 00000004:00000010:1.0:1773941414.259269:0:12018:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b885e1800. 00002000:00000010:1.0:1773941414.259272:0:12018:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b8800f000. 00000004:00000010:1.0:1773941414.259276:0:12018:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5b8801b800. 00000100:00000200:1.0:1773941414.259279:0:12018:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_cbd: registering portal 15 00000400:00000010:1.0:1773941414.259282:0:12018:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034d68. 00000400:00000010:1.0:1773941414.259286:0:12018:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b98115180. 00000100:00000010:1.0:1773941414.259293:0:12018:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b885e6000. 00000100:00000200:1.0:1773941414.259304:0:12018:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:3.0:1773941414.259316:0:12017:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b885b0000. 00000100:00000010:0.0:1773941414.259548:0:12019:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5ba039cf00. 00000020:00000010:0.0:1773941414.259553:0:12019:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5ba039cb00. 00000020:00000010:0.0:1773941414.259557:0:12019:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab8400. 00000020:00000010:0.0:1773941414.259560:0:12019:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab9c00. 00000040:00000010:0.0:1773941414.259563:0:12019:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab9600. 00000020:00000010:0.0:1773941414.259577:0:12019:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90db5000. 00000020:00000010:0.0:1773941414.259582:0:12019:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b9829eeb0. 00000020:00000010:0.0:1773941414.259585:0:12019:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab9a00. 80000000:00000010:0.0:1773941414.259588:0:12019:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b85ef6240. 40000000:00000010:0.0:1773941414.259591:0:12019:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5ba039c640. 00100000:00000010:0.0:1773941414.259594:0:12019:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b90db4000. 00040000:00000010:0.0:1773941414.259597:0:12019:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b99101700. 00040000:00000010:0.0:1773941414.259601:0:12019:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b90db6000. 00040000:00000010:0.0:1773941414.259604:0:12019:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab8800. 00000004:00000010:0.0:1773941414.259607:0:12019:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b90db3800. 00000004:00000010:0.0:1773941414.259611:0:12019:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b98aca000. 00000004:00000010:0.0:1773941414.259614:0:12019:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b90db6800. 00002000:00000010:0.0:1773941414.259617:0:12019:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0bf2400. 00000004:00000010:0.0:1773941414.259620:0:12019:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5ba0bf2800. 00000100:00000010:0.0:1773941414.259624:0:12019:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b90db2000. 00000100:00000200:0.0:1773941414.259636:0:12019:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:1.0:1773941414.259645:0:12017:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b862a5400. 00000100:00000010:3.0:1773941414.259798:0:12020:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5b85068240. 00000020:00000010:3.0:1773941414.259802:0:12020:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068cc0. 00000020:00000010:3.0:1773941414.259806:0:12020:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b885b1400. 00000020:00000010:3.0:1773941414.259808:0:12020:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b885b0e00. 00000040:00000010:3.0:1773941414.259810:0:12020:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b885b1200. 00000020:00000010:3.0:1773941414.259812:0:12020:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d97800. 00000020:00000010:3.0:1773941414.259815:0:12020:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b88498000. 00000020:00000010:3.0:1773941414.259816:0:12020:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b885b1a00. 80000000:00000010:3.0:1773941414.259818:0:12020:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 40000000:00000010:3.0:1773941414.259820:0:12020:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b85068800. 00100000:00000010:3.0:1773941414.259822:0:12020:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b90d91000. 00040000:00000010:3.0:1773941414.259824:0:12020:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b884bef00. 00040000:00000010:3.0:1773941414.259826:0:12020:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b90d92800. 00040000:00000010:3.0:1773941414.259828:0:12020:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b885b1c00. 00000004:00000010:3.0:1773941414.259834:0:12020:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b90d93000. 00000004:00000010:3.0:1773941414.259836:0:12020:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5ba0b8e000. 00000004:00000010:3.0:1773941414.259838:0:12020:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b90d95800. 00002000:00000010:3.0:1773941414.259840:0:12020:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0b93800. 00000004:00000010:3.0:1773941414.259842:0:12020:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5ba0b92000. 00000100:00000200:3.0:1773941414.259844:0:12020:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_cbd: registering portal 15 00000400:00000010:3.0:1773941414.259847:0:12020:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87e18. 00000400:00000010:3.0:1773941414.259849:0:12020:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b83ea3b40. 00000100:00000010:3.0:1773941414.259852:0:12020:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b90d91800. 00000100:00000200:3.0:1773941414.259869:0:12020:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:1.0:1773941414.259885:0:12017:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b862a4400. 00000100:00000010:3.0:1773941414.260017:0:12021:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5b85068940. 00000020:00000010:3.0:1773941414.260020:0:12021:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068600. 00000020:00000010:3.0:1773941414.260023:0:12021:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b885b0800. 00000020:00000010:3.0:1773941414.260024:0:12021:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b885b0c00. 00000040:00000010:3.0:1773941414.260026:0:12021:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b885b0200. 00000020:00000010:3.0:1773941414.260028:0:12021:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d90800. 00000020:00000010:3.0:1773941414.260031:0:12021:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b8849f3b8. 00000020:00000010:3.0:1773941414.260032:0:12021:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b885b1e00. 80000000:00000010:3.0:1773941414.260034:0:12021:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc92a0. 40000000:00000010:3.0:1773941414.260036:0:12021:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b85068700. 00100000:00000010:3.0:1773941414.260037:0:12021:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b90d94800. 00040000:00000010:3.0:1773941414.260039:0:12021:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b884be700. 00040000:00000010:3.0:1773941414.260041:0:12021:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b90d94000. 00040000:00000010:3.0:1773941414.260043:0:12021:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b9a587600. 00000004:00000010:3.0:1773941414.260045:0:12021:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b90d93800. 00000004:00000010:3.0:1773941414.260047:0:12021:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5ba0b89000. 00000004:00000010:3.0:1773941414.260049:0:12021:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b90d90000. 00002000:00000010:3.0:1773941414.260051:0:12021:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0b90400. 00000004:00000010:3.0:1773941414.260053:0:12021:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5ba0b90c00. 00000100:00000010:3.0:1773941414.260055:0:12021:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b90d95000. 00000100:00000200:3.0:1773941414.260063:0:12021:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:1.0:1773941414.260073:0:12017:0:(service.c:745:ptlrpc_register_service()) kmalloced '(service)': 360 at ffff8b5b98182400. 00000100:00000010:1.0:1773941414.260079:0:12017:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff8b5b88cb3000. 00000100:00000010:1.0:1773941414.260092:0:12017:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff8b5b850c0000. 00000100:00000010:1.0:1773941414.260099:0:12017:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff8b5b99127000. 00000100:00000010:1.0:1773941414.260103:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88cb1c00. 00000100:00000010:1.0:1773941414.260107:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d84000. 00000100:00000010:1.0:1773941414.260110:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88cb3800. 00000100:00000010:1.0:1773941414.260114:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d82000. 00000100:00000010:1.0:1773941414.260117:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0dc5800. 00000100:00000010:1.0:1773941414.260122:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b981a0000. 00000100:00000010:1.0:1773941414.260125:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0dc5400. 00000100:00000010:1.0:1773941414.260129:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b981a2000. 00000100:00000010:1.0:1773941414.260131:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0dc5c00. 00000100:00000010:1.0:1773941414.260134:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b981a6000. 00000100:00000010:1.0:1773941414.260136:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0dc4800. 00000100:00000010:1.0:1773941414.260139:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b83d0a000. 00000100:00000010:1.0:1773941414.260141:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8902ac00. 00000100:00000010:1.0:1773941414.260144:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b7c000. 00000100:00000010:1.0:1773941414.260150:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f3b800. 00000100:00000010:1.0:1773941414.260153:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b7e000. 00000100:00000010:1.0:1773941414.260155:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f39000. 00000100:00000010:1.0:1773941414.260158:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b78000. 00000100:00000010:1.0:1773941414.260159:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f39800. 00000100:00000010:1.0:1773941414.260162:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98af0000. 00000100:00000010:1.0:1773941414.260164:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f3bc00. 00000100:00000010:1.0:1773941414.260167:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98af2000. 00000100:00000010:1.0:1773941414.260168:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f38800. 00000100:00000010:1.0:1773941414.260171:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98af6000. 00000100:00000010:1.0:1773941414.260172:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f38400. 00000100:00000010:1.0:1773941414.260175:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98af4000. 00000100:00000010:1.0:1773941414.260176:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f3ac00. 00000100:00000010:1.0:1773941414.260180:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98aea000. 00000100:00000010:1.0:1773941414.260181:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f3a800. 00000100:00000010:1.0:1773941414.260184:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98aee000. 00000100:00000010:1.0:1773941414.260185:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f3a400. 00000100:00000010:1.0:1773941414.260188:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98aec000. 00000100:00000010:1.0:1773941414.260190:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b99ef4800. 00000100:00000010:1.0:1773941414.260193:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98ae8000. 00000100:00000010:1.0:1773941414.260194:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b99ef7400. 00000100:00000010:1.0:1773941414.260197:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b83cdc000. 00000100:00000010:1.0:1773941414.260199:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b99ef4000. 00000100:00000010:1.0:1773941414.260202:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b83cde000. 00000100:00000010:1.0:1773941414.260204:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b890c2000. 00000100:00000010:1.0:1773941414.260206:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b83cda000. 00000100:00000010:1.0:1773941414.260208:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b890c0800. 00000100:00000010:1.0:1773941414.260211:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b83cd8000. 00000100:00000010:1.0:1773941414.260213:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b890c3000. 00000100:00000010:1.0:1773941414.260217:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b829e2000. 00000100:00000010:1.0:1773941414.260219:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91e49000. 00000100:00000010:1.0:1773941414.260226:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b829e4000. 00000100:00000010:1.0:1773941414.260228:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91e4a400. 00000100:00000010:1.0:1773941414.260231:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b832ac000. 00000100:00000010:1.0:1773941414.260233:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91c4fc00. 00000100:00000010:1.0:1773941414.260236:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b832a8000. 00000100:00000010:1.0:1773941414.260237:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91c4d000. 00000100:00000010:1.0:1773941414.260240:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b832aa000. 00000100:00000010:1.0:1773941414.260241:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91c4d400. 00000100:00000010:1.0:1773941414.260244:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b91cac000. 00000100:00000010:1.0:1773941414.260246:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88011400. 00000100:00000010:1.0:1773941414.260249:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b91ca8000. 00000100:00000010:1.0:1773941414.260251:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88012000. 00000100:00000010:1.0:1773941414.260257:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98018000. 00000100:00000010:1.0:1773941414.260259:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88011c00. 00000100:00000010:1.0:1773941414.260262:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b9801c000. 00000100:00000010:1.0:1773941414.260263:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88012800. 00000100:00000010:1.0:1773941414.260266:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b9801e000. 00000100:00000010:1.0:1773941414.260267:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88011000. 00000100:00000010:1.0:1773941414.260270:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b9801a000. 00000100:00000010:1.0:1773941414.260271:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88010800. 00000100:00000010:1.0:1773941414.260277:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b850d0000. 00000100:00000010:1.0:1773941414.260279:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88012400. 00000100:00000010:1.0:1773941414.260282:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b850d4000. 00000100:00000010:1.0:1773941414.260283:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88011800. 00000100:00000010:1.0:1773941414.260286:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b850d6000. 00000100:00000010:1.0:1773941414.260288:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91f8e400. 00000100:00000010:1.0:1773941414.260291:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b850d2000. 00000100:00000010:1.0:1773941414.260293:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91f8f400. 00000100:00000010:1.0:1773941414.260300:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b9028c000. 00000100:00000010:1.0:1773941414.260302:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91f8e800. 00000100:00000010:1.0:1773941414.260306:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b9028e000. 00000100:00000010:1.0:1773941414.260310:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91f8cc00. 00000100:00000010:1.0:1773941414.260314:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b9028a000. 00000100:00000010:1.0:1773941414.260315:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91f8d400. 00000100:00000010:1.0:1773941414.260319:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90288000. 00000100:00000010:1.0:1773941414.260321:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91f8ec00. 00000100:00000010:1.0:1773941414.260327:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88934000. 00000100:00000010:1.0:1773941414.260329:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8801c800. 00000100:00000010:1.0:1773941414.260332:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88936000. 00000100:00000010:1.0:1773941414.260333:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85613800. 00000100:00000010:1.0:1773941414.260336:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88932000. 00000100:00000010:1.0:1773941414.260338:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85610c00. 00000100:00000010:1.0:1773941414.260340:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88930000. 00000100:00000010:1.0:1773941414.260342:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85611000. 00000100:00000010:1.0:1773941414.260348:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88cfc000. 00000100:00000010:1.0:1773941414.260350:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5bc2069400. 00000100:00000010:1.0:1773941414.260353:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88cfe000. 00000100:00000010:1.0:1773941414.260354:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5bc2069c00. 00000100:00000010:1.0:1773941414.260357:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88cfa000. 00000100:00000010:1.0:1773941414.260359:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5bc206b000. 00000100:00000010:1.0:1773941414.260361:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88cf8000. 00000100:00000010:1.0:1773941414.260363:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5bc2068000. 00000100:00000010:1.0:1773941414.260369:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b82c74000. 00000100:00000010:1.0:1773941414.260372:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8a3a2c00. 00000100:00000010:1.0:1773941414.260374:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b82c76000. 00000100:00000010:1.0:1773941414.260376:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5bc2076000. 00000100:00000010:1.0:1773941414.260379:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b82c72000. 00000100:00000010:1.0:1773941414.260382:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5bc2073400. 00000100:00000010:1.0:1773941414.260385:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b82c70000. 00000100:00000010:1.0:1773941414.260387:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5bc2072000. 00000100:00000010:1.0:1773941414.260406:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88940000. 00000100:00000010:1.0:1773941414.260411:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b99f55000. 00000100:00000010:1.0:1773941414.260419:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88944000. 00000100:00000010:1.0:1773941414.260422:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b99f54400. 00000100:00000010:1.0:1773941414.260426:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88946000. 00000100:00000010:1.0:1773941414.260429:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b99f57800. 00000100:00000010:1.0:1773941414.260432:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88942000. 00000100:00000010:1.0:1773941414.260435:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b99f56800. 00000100:00000010:1.0:1773941414.260442:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90da4000. 00000100:00000010:1.0:1773941414.260444:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91e15000. 00000100:00000010:1.0:1773941414.260447:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90da6000. 00000100:00000010:1.0:1773941414.260454:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b885acc00. 00000100:00000010:1.0:1773941414.260457:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90da2000. 00000100:00000010:1.0:1773941414.260459:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b885aec00. 00000100:00000010:1.0:1773941414.260461:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90da0000. 00000100:00000010:1.0:1773941414.260463:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b885adc00. 00000100:00000010:1.0:1773941414.260469:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b1e000. 00000100:00000010:1.0:1773941414.260471:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b885ad400. 00000100:00000010:1.0:1773941414.260474:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b1a000. 00000100:00000010:1.0:1773941414.260476:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b885afc00. 00000100:00000010:1.0:1773941414.260478:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b18000. 00000100:00000010:1.0:1773941414.260480:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b885ac800. 00000100:00000010:1.0:1773941414.260482:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b1c000. 00000100:00000010:1.0:1773941414.260484:0:12017:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff8b5b885ad800. 00000100:00000010:1.0:1773941414.260491:0:12017:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff8b5b893c4000. 00000100:00000010:1.0:1773941414.260496:0:12017:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff8b5b99123000. 00000100:00000010:1.0:1773941414.260498:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b885af800. 00000100:00000010:1.0:1773941414.260504:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a12000. 00000100:00000010:1.0:1773941414.260506:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b885ac400. 00000100:00000010:1.0:1773941414.260508:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a10000. 00000100:00000010:1.0:1773941414.260510:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b885ad000. 00000100:00000010:1.0:1773941414.260512:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a14000. 00000100:00000010:1.0:1773941414.260514:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b885ae400. 00000100:00000010:1.0:1773941414.260519:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a16000. 00000100:00000010:1.0:1773941414.260521:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b885ae800. 00000100:00000010:1.0:1773941414.260527:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d6c000. 00000100:00000010:1.0:1773941414.260529:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b885ae000. 00000100:00000010:1.0:1773941414.260532:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d6e000. 00000100:00000010:1.0:1773941414.260533:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b885af400. 00000100:00000010:1.0:1773941414.260536:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d6a000. 00000100:00000010:1.0:1773941414.260538:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b885ac000. 00000100:00000010:1.0:1773941414.260540:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d68000. 00000100:00000010:1.0:1773941414.260542:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b885af000. 00000100:00000010:1.0:1773941414.260548:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a3a000. 00000100:00000010:1.0:1773941414.260555:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90237c00. 00000100:00000010:1.0:1773941414.260558:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a38000. 00000100:00000010:1.0:1773941414.260559:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90234800. 00000100:00000010:1.0:1773941414.260562:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a3c000. 00000100:00000010:1.0:1773941414.260563:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90235800. 00000100:00000010:1.0:1773941414.260566:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a3e000. 00000100:00000010:1.0:1773941414.260568:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90237800. 00000100:00000010:1.0:1773941414.260573:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88994000. 00000100:00000010:1.0:1773941414.260575:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90234400. 00000100:00000010:1.0:1773941414.260577:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88996000. 00000100:00000010:1.0:1773941414.260579:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90235000. 00000100:00000010:1.0:1773941414.260581:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88992000. 00000100:00000010:1.0:1773941414.260583:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90236400. 00000100:00000010:1.0:1773941414.260586:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88990000. 00000100:00000010:1.0:1773941414.260587:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90236800. 00000100:00000010:1.0:1773941414.260593:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889f2000. 00000100:00000010:1.0:1773941414.260595:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90236000. 00000100:00000010:1.0:1773941414.260598:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889f0000. 00000100:00000010:1.0:1773941414.260599:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90237400. 00000100:00000010:1.0:1773941414.260602:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889f4000. 00000100:00000010:1.0:1773941414.260606:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90234000. 00000100:00000010:1.0:1773941414.260609:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889f6000. 00000100:00000010:1.0:1773941414.260612:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90237000. 00000100:00000010:1.0:1773941414.260618:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b8505a000. 00000100:00000010:1.0:1773941414.260620:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90234c00. 00000100:00000010:1.0:1773941414.260623:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b85058000. 00000100:00000010:1.0:1773941414.260624:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90236c00. 00000100:00000010:1.0:1773941414.260627:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b8505c000. 00000100:00000010:1.0:1773941414.260629:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90235c00. 00000100:00000010:1.0:1773941414.260631:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b8505e000. 00000100:00000010:1.0:1773941414.260633:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90235400. 00000100:00000010:1.0:1773941414.260640:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b9022a000. 00000100:00000010:1.0:1773941414.260645:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85071c00. 00000100:00000010:1.0:1773941414.260650:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90228000. 00000100:00000010:1.0:1773941414.260651:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85071400. 00000100:00000010:1.0:1773941414.260654:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b9022c000. 00000100:00000010:1.0:1773941414.260656:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85073c00. 00000100:00000010:1.0:1773941414.260659:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b9022e000. 00000100:00000010:1.0:1773941414.260661:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85070800. 00000100:00000010:1.0:1773941414.260667:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885f0000. 00000100:00000010:1.0:1773941414.260668:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85071800. 00000100:00000010:1.0:1773941414.260671:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885f4000. 00000100:00000010:1.0:1773941414.260673:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85073800. 00000100:00000010:1.0:1773941414.260675:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885f6000. 00000100:00000010:1.0:1773941414.260677:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85070400. 00000100:00000010:1.0:1773941414.260680:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885f2000. 00000100:00000010:1.0:1773941414.260681:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85071000. 00000100:00000010:1.0:1773941414.260689:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90246000. 00000100:00000010:1.0:1773941414.260691:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85072400. 00000100:00000010:1.0:1773941414.260694:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90242000. 00000100:00000010:1.0:1773941414.260696:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85072800. 00000100:00000010:1.0:1773941414.260835:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90240000. 00000100:00000010:1.0:1773941414.260966:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85072000. 00000100:00000010:1.0:1773941414.260972:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90244000. 00000100:00000010:1.0:1773941414.260975:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85073400. 00000100:00000010:1.0:1773941414.260985:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90254000. 00000100:00000010:1.0:1773941414.260987:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85070000. 00000100:00000010:1.0:1773941414.260991:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90256000. 00000100:00000010:1.0:1773941414.260994:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85073000. 00000100:00000010:1.0:1773941414.260998:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90252000. 00000100:00000010:1.0:1773941414.261000:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85070c00. 00000100:00000010:1.0:1773941414.261004:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90250000. 00000100:00000010:1.0:1773941414.261006:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85072c00. 00000100:00000010:1.0:1773941414.261014:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889ce000. 00000100:00000010:1.0:1773941414.261021:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889ec400. 00000100:00000010:1.0:1773941414.261024:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889ca000. 00000100:00000010:1.0:1773941414.261026:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889ed000. 00000100:00000010:1.0:1773941414.261029:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889c8000. 00000100:00000010:1.0:1773941414.261031:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889ee400. 00000100:00000010:1.0:1773941414.261034:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889cc000. 00000100:00000010:1.0:1773941414.261036:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889ee800. 00000100:00000010:1.0:1773941414.261042:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d4e000. 00000100:00000010:1.0:1773941414.261045:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889ee000. 00000100:00000010:1.0:1773941414.261048:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d4a000. 00000100:00000010:1.0:1773941414.261050:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889ef400. 00000100:00000010:1.0:1773941414.261053:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d48000. 00000100:00000010:1.0:1773941414.261054:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889ec000. 00000100:00000010:1.0:1773941414.261057:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d4c000. 00000100:00000010:1.0:1773941414.261058:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889ef000. 00000100:00000010:1.0:1773941414.261064:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90df2000. 00000100:00000010:1.0:1773941414.261066:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889ecc00. 00000100:00000010:1.0:1773941414.261068:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90df0000. 00000100:00000010:1.0:1773941414.261074:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889eec00. 00000100:00000010:1.0:1773941414.261078:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90df4000. 00000100:00000010:1.0:1773941414.261081:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889edc00. 00000100:00000010:1.0:1773941414.261085:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90df6000. 00000100:00000010:1.0:1773941414.261087:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889ed400. 00000100:00000010:1.0:1773941414.261097:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d12000. 00000100:00000010:1.0:1773941414.261100:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889efc00. 00000100:00000010:1.0:1773941414.261104:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d10000. 00000100:00000010:1.0:1773941414.261107:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889ec800. 00000100:00000010:1.0:1773941414.261111:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d14000. 00000100:00000010:1.0:1773941414.261114:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889ed800. 00000100:00000010:1.0:1773941414.261118:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d16000. 00000100:00000010:1.0:1773941414.261121:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b889ef800. 00000100:00000010:1.0:1773941414.261130:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885d2000. 00000100:00000010:1.0:1773941414.261136:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0c29000. 00000100:00000010:1.0:1773941414.261140:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885d0000. 00000100:00000010:1.0:1773941414.261142:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0c2a400. 00000100:00000010:1.0:1773941414.261144:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885d4000. 00000100:00000010:1.0:1773941414.261147:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0c2a800. 00000100:00000010:1.0:1773941414.261150:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885d6000. 00000100:00000010:1.0:1773941414.261151:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0c2a000. 00000100:00000010:1.0:1773941414.261157:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0ca4000. 00000100:00000010:1.0:1773941414.261159:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0c2b400. 00000100:00000010:1.0:1773941414.261161:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0ca6000. 00000100:00000010:1.0:1773941414.261163:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0c28000. 00000100:00000010:1.0:1773941414.261166:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0ca2000. 00000100:00000010:1.0:1773941414.261167:0:12017:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0c2b000. 00000100:00000010:1.0:1773941414.261169:0:12017:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0ca0000. 00000400:00000200:1.0:1773941414.261171:0:12017:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 17 lazy 00000020:00000010:1.0:1773941414.261195:0:12017:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5ba0c28c00. 00000020:00000010:1.0:1773941414.261199:0:12017:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at ffff8b5b85e58000. 00000100:00000010:1.0:1773941414.261228:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b98115e40. 00000100:00000010:1.0:1773941414.261230:0:12017:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff8b5b82c59740. 00000100:00000010:1.0:1773941414.261233:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b98115540. 00000100:00000010:1.0:1773941414.261235:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b98115d80. 00000100:00000010:1.0:1773941414.261237:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b91fc0780. 00000100:00000010:1.0:1773941414.261239:0:12017:0:(nrs.c:1026:nrs_svcpt_setup_locked()) kmalloced '(svcpt->scp_nrs_hp)': 112 at ffff8b5b90edb400. 00000100:00000010:1.0:1773941414.261241:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b91fc0e40. 00000100:00000010:1.0:1773941414.261242:0:12017:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff8b5b82c59100. 00000100:00000010:1.0:1773941414.261244:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b91fc0900. 00000100:00000010:1.0:1773941414.261245:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b91fc0cc0. 00000100:00000010:1.0:1773941414.261246:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b850fa180. 00000100:00000010:1.0:1773941414.261248:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b850fad80. 00000100:00000010:1.0:1773941414.261249:0:12017:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff8b5b82c59300. 00000100:00000010:1.0:1773941414.261251:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b850faa80. 00000100:00000010:1.0:1773941414.261252:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b850fa3c0. 00000100:00000010:1.0:1773941414.261253:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b850facc0. 00000100:00000010:1.0:1773941414.261254:0:12017:0:(nrs.c:1026:nrs_svcpt_setup_locked()) kmalloced '(svcpt->scp_nrs_hp)': 112 at ffff8b5b90edbe80. 00000100:00000010:1.0:1773941414.261256:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b850fac00. 00000100:00000010:1.0:1773941414.261257:0:12017:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff8b5b82c59840. 00000100:00000010:1.0:1773941414.261258:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b850fa300. 00000100:00000010:1.0:1773941414.261259:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b850fa9c0. 00000100:00000010:1.0:1773941414.261260:0:12017:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b850fa000. 00000100:00000200:1.0:1773941414.261279:0:12017:0:(service.c:848:ptlrpc_register_service()) ldlm_canceld: Started, listening on portal 17 00000100:00000010:1.0:1773941414.261281:0:12017:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b98183400. 00000100:00000010:0.0:1773941414.261480:0:12022:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5ba039c3c0. 00000020:00000010:0.0:1773941414.261485:0:12022:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5ba039ca00. 00000020:00000010:0.0:1773941414.261489:0:12022:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab8000. 00000020:00000010:0.0:1773941414.261491:0:12022:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab9800. 00000020:00000010:0.0:1773941414.261495:0:12022:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5b85ef6de0. 00000040:00000010:0.0:1773941414.261497:0:12022:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab9e00. 00000020:00000010:0.0:1773941414.261500:0:12022:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90db4800. 00000020:00000010:0.0:1773941414.261504:0:12022:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b9829e4a0. 00000020:00000010:0.0:1773941414.261507:0:12022:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab8200. 80000000:00000010:0.0:1773941414.261509:0:12022:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b85ef6300. 40000000:00000010:0.0:1773941414.261517:0:12022:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5ba039cd40. 00008000:00000010:0.0:1773941414.261522:0:12022:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90283c08. 00100000:00000010:0.0:1773941414.261525:0:12022:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b90db7800. 00040000:00000010:0.0:1773941414.261528:0:12022:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b99101c00. 00040000:00000010:0.0:1773941414.261530:0:12022:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b90db3000. 00040000:00000010:0.0:1773941414.261533:0:12022:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab9000. 00000004:00000010:0.0:1773941414.261536:0:12022:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b90db7000. 00000004:00000010:0.0:1773941414.261539:0:12022:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b98acb000. 00000004:00000010:0.0:1773941414.261542:0:12022:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b90db5800. 00002000:00000010:0.0:1773941414.261545:0:12022:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0bf0800. 00000004:00000010:0.0:1773941414.261556:0:12022:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5ba0bf2000. 00000100:00000200:0.0:1773941414.261560:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261564:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014bb58. 00000400:00000010:0.0:1773941414.261567:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f8a80. 00000100:00000200:0.0:1773941414.261570:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261572:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014bf20. 00000400:00000010:0.0:1773941414.261574:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f8d80. 00000100:00000200:0.0:1773941414.261576:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261578:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014b580. 00000400:00000010:0.0:1773941414.261580:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f8480. 00000100:00000200:0.0:1773941414.261582:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261584:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014b898. 00000400:00000010:0.0:1773941414.261586:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f8f00. 00000100:00000200:0.0:1773941414.261588:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261590:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014be70. 00000400:00000010:0.0:1773941414.261592:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f8000. 00000100:00000200:0.0:1773941414.261595:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261597:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014b9a0. 00000400:00000010:0.0:1773941414.261599:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f86c0. 00000100:00000200:0.0:1773941414.261601:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261603:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014b000. 00000400:00000010:0.0:1773941414.261605:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f8cc0. 00000100:00000200:0.0:1773941414.261607:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261609:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014b5d8. 00000400:00000010:0.0:1773941414.261611:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f8540. 00000100:00000200:0.0:1773941414.261619:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261622:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014b8f0. 00000400:00000010:0.0:1773941414.261625:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f8840. 00000100:00000200:0.0:1773941414.261627:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261628:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014b840. 00000400:00000010:0.0:1773941414.261631:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f8240. 00000100:00000200:0.0:1773941414.261633:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261635:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014bdc0. 00000400:00000010:0.0:1773941414.261637:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f8900. 00000100:00000200:0.0:1773941414.261639:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261641:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8014bec8. 00000400:00000010:0.0:1773941414.261644:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f83c0. 00000100:00000200:0.0:1773941414.261646:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261658:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7058. 00000400:00000010:0.0:1773941414.261661:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f8c00. 00000100:00000200:0.0:1773941414.261664:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261666:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7e18. 00000400:00000010:0.0:1773941414.261669:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f8600. 00000100:00000200:0.0:1773941414.261671:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261673:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7aa8. 00000400:00000010:0.0:1773941414.261675:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f89c0. 00000100:00000200:0.0:1773941414.261677:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261679:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7c60. 00000400:00000010:0.0:1773941414.261682:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f8780. 00000100:00000200:0.0:1773941414.261684:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261685:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7210. 00000400:00000010:0.0:1773941414.261688:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b889f8b40. 00000100:00000200:0.0:1773941414.261690:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261693:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b73c8. 00000400:00000010:0.0:1773941414.261697:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ede3c0. 00000100:00000200:0.0:1773941414.261699:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261701:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b79a0. 00000400:00000010:0.0:1773941414.261722:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85edeb40. 00000100:00000200:0.0:1773941414.261726:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261729:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b77e8. 00000400:00000010:0.0:1773941414.261736:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85eded80. 00000100:00000200:0.0:1773941414.261739:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261741:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7ec8. 00000400:00000010:0.0:1773941414.261743:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85edea80. 00000100:00000200:0.0:1773941414.261746:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261747:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7dc0. 00000400:00000010:0.0:1773941414.261750:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ede780. 00000100:00000200:0.0:1773941414.261751:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261753:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7840. 00000400:00000010:0.0:1773941414.261755:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ede0c0. 00000100:00000200:0.0:1773941414.261757:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261759:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b78f0. 00000400:00000010:0.0:1773941414.261760:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85edef00. 00000100:00000200:0.0:1773941414.261762:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261764:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7000. 00000400:00000010:0.0:1773941414.261766:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ede240. 00000100:00000200:0.0:1773941414.261768:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261770:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b75d8. 00000400:00000010:0.0:1773941414.261772:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ede600. 00000100:00000200:0.0:1773941414.261774:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261776:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7b58. 00000400:00000010:0.0:1773941414.261778:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ede840. 00000100:00000200:0.0:1773941414.261780:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261782:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7f20. 00000400:00000010:0.0:1773941414.261784:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ede6c0. 00000100:00000200:0.0:1773941414.261786:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261788:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7580. 00000400:00000010:0.0:1773941414.261790:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ede300. 00000100:00000200:0.0:1773941414.261793:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261795:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7898. 00000400:00000010:0.0:1773941414.261797:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ede180. 00000100:00000200:0.0:1773941414.261799:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261800:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7e70. 00000400:00000010:0.0:1773941414.261803:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b98124300. 00000100:00000200:0.0:1773941414.261805:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261812:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7370. 00000400:00000010:0.0:1773941414.261815:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b98124cc0. 00000100:00000200:0.0:1773941414.261818:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261820:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7268. 00000400:00000010:0.0:1773941414.261822:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b98124780. 00000100:00000200:0.0:1773941414.261824:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261826:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7bb0. 00000400:00000010:0.0:1773941414.261828:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b98124a80. 00000100:00000200:0.0:1773941414.261830:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261832:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7cb8. 00000400:00000010:0.0:1773941414.261833:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b98124f00. 00000100:00000200:0.0:1773941414.261835:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261837:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7738. 00000400:00000010:0.0:1773941414.261840:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e6c0. 00000100:00000200:0.0:1773941414.261842:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261844:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7790. 00000400:00000010:0.0:1773941414.261846:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5ec00. 00000100:00000200:0.0:1773941414.261848:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261850:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7108. 00000400:00000010:0.0:1773941414.261852:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5ef00. 00000100:00000200:0.0:1773941414.261854:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261856:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7630. 00000400:00000010:0.0:1773941414.261859:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e780. 00000100:00000200:0.0:1773941414.261860:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261862:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7b00. 00000400:00000010:0.0:1773941414.261864:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5ee40. 00000100:00000200:0.0:1773941414.261866:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261868:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b79f8. 00000400:00000010:0.0:1773941414.261870:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e3c0. 00000100:00000200:0.0:1773941414.261873:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261874:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7d10. 00000400:00000010:0.0:1773941414.261876:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e000. 00000100:00000200:0.0:1773941414.261878:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261879:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7d68. 00000400:00000010:0.0:1773941414.261881:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e180. 00000100:00000200:0.0:1773941414.261889:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261892:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7948. 00000400:00000010:0.0:1773941414.261894:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e540. 00000100:00000200:0.0:1773941414.261896:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261898:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7c08. 00000400:00000010:0.0:1773941414.261900:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e840. 00000100:00000200:0.0:1773941414.261902:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261904:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7a50. 00000400:00000010:0.0:1773941414.261906:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5ea80. 00000100:00000200:0.0:1773941414.261909:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261911:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7478. 00000400:00000010:0.0:1773941414.261914:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5ecc0. 00000100:00000200:0.0:1773941414.261916:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261918:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b71b8. 00000400:00000010:0.0:1773941414.261920:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5eb40. 00000100:00000200:0.0:1773941414.261922:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261924:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7318. 00000400:00000010:0.0:1773941414.261926:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e900. 00000100:00000200:0.0:1773941414.261928:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261930:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7528. 00000400:00000010:0.0:1773941414.261932:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5ed80. 00000100:00000200:0.0:1773941414.261934:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261935:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7160. 00000400:00000010:0.0:1773941414.261938:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e480. 00000100:00000200:0.0:1773941414.261939:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261941:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7f78. 00000400:00000010:0.0:1773941414.261943:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e600. 00000100:00000200:0.0:1773941414.261945:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261947:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7420. 00000400:00000010:0.0:1773941414.261950:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e9c0. 00000100:00000200:0.0:1773941414.261952:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261954:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b72c0. 00000400:00000010:0.0:1773941414.261956:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e0c0. 00000100:00000200:0.0:1773941414.261958:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261960:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b70b0. 00000400:00000010:0.0:1773941414.261967:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e300. 00000100:00000200:0.0:1773941414.261969:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261971:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b7688. 00000400:00000010:0.0:1773941414.261975:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84476d80. 00000100:00000200:0.0:1773941414.261977:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261979:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b74d0. 00000400:00000010:0.0:1773941414.261982:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b89064780. 00000100:00000200:0.0:1773941414.261984:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.261986:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b885b76e0. 00000400:00000010:0.0:1773941414.261989:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b89064c00. 00000100:00000200:0.0:1773941414.261991:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.262001:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93108. 00000400:00000010:0.0:1773941414.262003:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b89064a80. 00000100:00000200:0.0:1773941414.262006:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.262008:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93630. 00000400:00000010:0.0:1773941414.262010:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b89064900. 00000100:00000200:0.0:1773941414.262013:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.262015:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93b00. 00000400:00000010:0.0:1773941414.262017:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b89064240. 00000100:00000200:0.0:1773941414.262019:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.262021:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c939f8. 00000400:00000010:0.0:1773941414.262024:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b89064e40. 00000100:00000200:0.0:1773941414.262026:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.262028:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93d10. 00000400:00000010:0.0:1773941414.262030:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b89064600. 00000100:00000200:0.0:1773941414.262033:0:12022:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941414.262035:0:12022:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93d68. 00000400:00000010:0.0:1773941414.262038:0:12022:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b890646c0. 00000100:00000010:0.0:1773941414.262042:0:12022:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b90db1800. 00000100:00000200:0.0:1773941414.262057:0:12022:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:1.0:1773941414.262079:0:12017:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b98183c00. 00000100:00000010:1.0:1773941414.262249:0:12023:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5b82c599c0. 00000020:00000010:1.0:1773941414.262252:0:12023:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b82c59540. 00000020:00000010:1.0:1773941414.262254:0:12023:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b839e5e00. 00000020:00000010:1.0:1773941414.262257:0:12023:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b91d90000. 00000020:00000010:1.0:1773941414.262258:0:12023:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5b84cfb1e0. 00000040:00000010:1.0:1773941414.262261:0:12023:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b91d91800. 00000020:00000010:1.0:1773941414.262263:0:12023:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b99ed1800. 00000020:00000010:1.0:1773941414.262266:0:12023:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b98a04670. 00000020:00000010:1.0:1773941414.262274:0:12023:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b91d90400. 80000000:00000010:1.0:1773941414.262277:0:12023:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b84cfb9c0. 40000000:00000010:1.0:1773941414.262279:0:12023:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b82c59400. 00008000:00000010:1.0:1773941414.262282:0:12023:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d26a60. 00100000:00000010:1.0:1773941414.262286:0:12023:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b88952800. 00040000:00000010:1.0:1773941414.262288:0:12023:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5ba0c22700. 00040000:00000010:1.0:1773941414.262291:0:12023:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b88952000. 00040000:00000010:1.0:1773941414.262295:0:12023:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b8a0d1800. 00000004:00000010:1.0:1773941414.262297:0:12023:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b88957000. 00000004:00000010:1.0:1773941414.262301:0:12023:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b85e5b000. 00000004:00000010:1.0:1773941414.262304:0:12023:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b88955800. 00002000:00000010:1.0:1773941414.262307:0:12023:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0c2ac00. 00000004:00000010:1.0:1773941414.262310:0:12023:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5ba0c29c00. 00000100:00000010:1.0:1773941414.262313:0:12023:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b88953000. 00000100:00000200:1.0:1773941414.262326:0:12023:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:3.0:1773941414.262341:0:12017:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b845d5c00. 00000100:00000010:0.0:1773941414.262525:0:12024:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5ba039c100. 00000020:00000010:0.0:1773941414.262529:0:12024:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5ba039cfc0. 00000020:00000010:0.0:1773941414.262534:0:12024:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b91fe9200. 00000020:00000010:0.0:1773941414.262537:0:12024:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b8a0f2400. 00000020:00000010:0.0:1773941414.262540:0:12024:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5b85ef6000. 00000040:00000010:0.0:1773941414.262550:0:12024:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b893cfe00. 00000020:00000010:0.0:1773941414.262553:0:12024:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90db0800. 00000020:00000010:0.0:1773941414.262556:0:12024:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b98298f18. 00000020:00000010:0.0:1773941414.262558:0:12024:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b893cfc00. 80000000:00000010:0.0:1773941414.262562:0:12024:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b85ef6420. 40000000:00000010:0.0:1773941414.262564:0:12024:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5ba039ce80. 00008000:00000010:0.0:1773941414.262567:0:12024:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90283880. 00100000:00000010:0.0:1773941414.262570:0:12024:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b90db0000. 00040000:00000010:0.0:1773941414.262573:0:12024:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b99101b00. 00040000:00000010:0.0:1773941414.262577:0:12024:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b86885800. 00040000:00000010:0.0:1773941414.262579:0:12024:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b893cfa00. 00000004:00000010:0.0:1773941414.262582:0:12024:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b8a327000. 00000004:00000010:0.0:1773941414.262586:0:12024:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b98ac8000. 00000004:00000010:0.0:1773941414.262589:0:12024:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b8a324800. 00002000:00000010:0.0:1773941414.262592:0:12024:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0bf1400. 00000004:00000010:0.0:1773941414.262594:0:12024:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5ba0bf1c00. 00000100:00000010:0.0:1773941414.262598:0:12024:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5ba0b0d800. 00000100:00000200:0.0:1773941414.262611:0:12024:0:(service.c:3086:ptlrpc_main()) service thread 2 (#3) started 00000100:00000010:1.0:1773941414.262616:0:12017:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b83991e00. 00000100:00000010:2.0:1773941414.262805:0:12025:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5b845add40. 00000020:00000010:2.0:1773941414.262809:0:12025:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b845add00. 00000020:00000010:2.0:1773941414.262813:0:12025:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84fef600. 00000020:00000010:2.0:1773941414.262816:0:12025:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b84fee800. 00000020:00000010:2.0:1773941414.262819:0:12025:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0166240. 00000040:00000010:2.0:1773941414.262821:0:12025:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b84feee00. 00000020:00000010:2.0:1773941414.262824:0:12025:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90f27800. 00000020:00000010:2.0:1773941414.262828:0:12025:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b88558000. 00000020:00000010:2.0:1773941414.262831:0:12025:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b84fee400. 80000000:00000010:2.0:1773941414.262833:0:12025:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba01664e0. 40000000:00000010:2.0:1773941414.262836:0:12025:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b845ad1c0. 00008000:00000010:2.0:1773941414.262839:0:12025:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d774f8. 00100000:00000010:2.0:1773941414.262842:0:12025:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b90f24000. 00040000:00000010:2.0:1773941414.262844:0:12025:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98bb0400. 00040000:00000010:2.0:1773941414.262847:0:12025:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b90f24800. 00040000:00000010:2.0:1773941414.262850:0:12025:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b84fee000. 00000004:00000010:2.0:1773941414.262853:0:12025:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b90f25000. 00000004:00000010:2.0:1773941414.262856:0:12025:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b88905000. 00000004:00000010:2.0:1773941414.262859:0:12025:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b90f23800. 00002000:00000010:2.0:1773941414.262861:0:12025:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b83f0ac00. 00000004:00000010:2.0:1773941414.262864:0:12025:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5b83f0b000. 00000100:00000200:2.0:1773941414.262867:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262870:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab840. 00000400:00000010:2.0:1773941414.262874:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90d71540. 00000100:00000200:2.0:1773941414.262877:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262884:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab8f0. 00000400:00000010:2.0:1773941414.262887:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90d71240. 00000100:00000200:2.0:1773941414.262889:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262891:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab000. 00000400:00000010:2.0:1773941414.262893:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90d710c0. 00000100:00000200:2.0:1773941414.262895:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262897:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab5d8. 00000400:00000010:2.0:1773941414.262899:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90d71840. 00000100:00000200:2.0:1773941414.262901:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262903:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abb58. 00000400:00000010:2.0:1773941414.262905:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90d71e40. 00000100:00000200:2.0:1773941414.262907:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262909:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abf20. 00000400:00000010:2.0:1773941414.262910:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90d71000. 00000100:00000200:2.0:1773941414.262912:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262914:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab580. 00000400:00000010:2.0:1773941414.262916:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90d71f00. 00000100:00000200:2.0:1773941414.262917:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262919:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab898. 00000400:00000010:2.0:1773941414.262921:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90d71cc0. 00000100:00000200:2.0:1773941414.262923:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262925:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abe70. 00000400:00000010:2.0:1773941414.262927:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90d71600. 00000100:00000200:2.0:1773941414.262928:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262929:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab370. 00000400:00000010:2.0:1773941414.262930:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90d71900. 00000100:00000200:2.0:1773941414.262932:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262933:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab268. 00000400:00000010:2.0:1773941414.262934:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90d716c0. 00000100:00000200:2.0:1773941414.262935:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262936:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abbb0. 00000400:00000010:2.0:1773941414.262937:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90d71a80. 00000100:00000200:2.0:1773941414.262938:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262939:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abcb8. 00000400:00000010:2.0:1773941414.262941:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90d71180. 00000100:00000200:2.0:1773941414.262946:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262947:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab738. 00000400:00000010:2.0:1773941414.262949:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90d71c00. 00000100:00000200:2.0:1773941414.262950:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262951:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab790. 00000400:00000010:2.0:1773941414.262953:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90d719c0. 00000100:00000200:2.0:1773941414.262954:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262956:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab108. 00000400:00000010:2.0:1773941414.262957:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b88ca6cc0. 00000100:00000200:2.0:1773941414.262958:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262959:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab630. 00000400:00000010:2.0:1773941414.262961:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed70c0. 00000100:00000200:2.0:1773941414.262962:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262963:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abb00. 00000400:00000010:2.0:1773941414.262964:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7480. 00000100:00000200:2.0:1773941414.262965:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262966:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab9f8. 00000400:00000010:2.0:1773941414.262968:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7540. 00000100:00000200:2.0:1773941414.262969:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262970:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abd10. 00000400:00000010:2.0:1773941414.262971:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7780. 00000100:00000200:2.0:1773941414.262972:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262973:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abd68. 00000400:00000010:2.0:1773941414.262974:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7d80. 00000100:00000200:2.0:1773941414.262975:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262976:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab948. 00000400:00000010:2.0:1773941414.262977:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7180. 00000100:00000200:2.0:1773941414.262978:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262979:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abc08. 00000400:00000010:2.0:1773941414.262980:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7c00. 00000100:00000200:2.0:1773941414.262981:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262982:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845aba50. 00000400:00000010:2.0:1773941414.262983:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7300. 00000100:00000200:2.0:1773941414.262984:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262985:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab478. 00000400:00000010:2.0:1773941414.262989:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7240. 00000100:00000200:2.0:1773941414.262991:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262992:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab1b8. 00000400:00000010:2.0:1773941414.262994:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7b40. 00000100:00000200:2.0:1773941414.262995:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262996:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab318. 00000400:00000010:2.0:1773941414.262997:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7840. 00000100:00000200:2.0:1773941414.262998:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.262999:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab528. 00000400:00000010:2.0:1773941414.263000:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7000. 00000100:00000200:2.0:1773941414.263001:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263002:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab160. 00000400:00000010:2.0:1773941414.263003:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed79c0. 00000100:00000200:2.0:1773941414.263004:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263005:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abf78. 00000400:00000010:2.0:1773941414.263006:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7e40. 00000100:00000200:2.0:1773941414.263007:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263008:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab420. 00000400:00000010:2.0:1773941414.263009:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7f00. 00000100:00000200:2.0:1773941414.263010:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263011:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab2c0. 00000400:00000010:2.0:1773941414.263012:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7900. 00000100:00000200:2.0:1773941414.263013:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263014:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab0b0. 00000400:00000010:2.0:1773941414.263015:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed76c0. 00000100:00000200:2.0:1773941414.263016:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263017:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab688. 00000400:00000010:2.0:1773941414.263018:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7cc0. 00000100:00000200:2.0:1773941414.263019:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263020:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab4d0. 00000400:00000010:2.0:1773941414.263021:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7600. 00000100:00000200:2.0:1773941414.263022:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263023:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:2.0:1773941414.263024:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed73c0. 00000100:00000200:2.0:1773941414.263025:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263029:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab058. 00000400:00000010:2.0:1773941414.263030:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7a80. 00000100:00000200:2.0:1773941414.263032:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263033:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abe18. 00000400:00000010:2.0:1773941414.263034:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b480. 00000100:00000200:2.0:1773941414.263036:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263036:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abaa8. 00000400:00000010:2.0:1773941414.263038:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680bb40. 00000100:00000200:2.0:1773941414.263039:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263040:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abc60. 00000400:00000010:2.0:1773941414.263041:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b780. 00000100:00000200:2.0:1773941414.263042:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263043:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab210. 00000400:00000010:2.0:1773941414.263044:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b180. 00000100:00000200:2.0:1773941414.263045:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263046:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abdc0. 00000400:00000010:2.0:1773941414.263047:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b0c0. 00000100:00000200:2.0:1773941414.263048:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263049:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abec8. 00000400:00000010:2.0:1773941414.263051:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680be40. 00000100:00000200:2.0:1773941414.263051:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263052:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab7e8. 00000400:00000010:2.0:1773941414.263053:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b3c0. 00000100:00000200:2.0:1773941414.263054:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263055:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab9a0. 00000400:00000010:2.0:1773941414.263057:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680bcc0. 00000100:00000200:2.0:1773941414.263058:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263059:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab3c8. 00000400:00000010:2.0:1773941414.263060:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680bf00. 00000100:00000200:2.0:1773941414.263061:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263062:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9a50. 00000400:00000010:2.0:1773941414.263063:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680bc00. 00000100:00000200:2.0:1773941414.263064:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263065:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9478. 00000400:00000010:2.0:1773941414.263067:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b9c0. 00000100:00000200:2.0:1773941414.263070:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263072:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af91b8. 00000400:00000010:2.0:1773941414.263073:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b900. 00000100:00000200:2.0:1773941414.263074:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263075:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9318. 00000400:00000010:2.0:1773941414.263076:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b240. 00000100:00000200:2.0:1773941414.263077:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263078:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9528. 00000400:00000010:2.0:1773941414.263079:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680ba80. 00000100:00000200:2.0:1773941414.263080:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263081:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9160. 00000400:00000010:2.0:1773941414.263082:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b300. 00000100:00000200:2.0:1773941414.263083:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263084:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9f78. 00000400:00000010:2.0:1773941414.263086:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0cfba80. 00000100:00000200:2.0:1773941414.263087:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263088:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9420. 00000400:00000010:2.0:1773941414.263089:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0cfbcc0. 00000100:00000200:2.0:1773941414.263090:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263091:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af92c0. 00000400:00000010:2.0:1773941414.263092:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0cfbd80. 00000100:00000200:2.0:1773941414.263093:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263094:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af90b0. 00000400:00000010:2.0:1773941414.263095:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0cfbb40. 00000100:00000200:2.0:1773941414.263096:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263097:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9688. 00000400:00000010:2.0:1773941414.263098:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0cfbe40. 00000100:00000200:2.0:1773941414.263099:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263100:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af94d0. 00000400:00000010:2.0:1773941414.263101:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0cfb000. 00000100:00000200:2.0:1773941414.263110:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263116:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af96e0. 00000400:00000010:2.0:1773941414.263118:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0cfb300. 00000100:00000200:2.0:1773941414.263120:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263122:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9058. 00000400:00000010:2.0:1773941414.263130:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0cfb240. 00000100:00000200:2.0:1773941414.263132:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263135:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9e18. 00000400:00000010:2.0:1773941414.263137:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0cfb600. 00000100:00000200:2.0:1773941414.263139:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263141:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9aa8. 00000400:00000010:2.0:1773941414.263143:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0cfb840. 00000100:00000200:2.0:1773941414.263145:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263147:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9c60. 00000400:00000010:2.0:1773941414.263149:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0cfb540. 00000100:00000200:2.0:1773941414.263151:0:12025:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941414.263153:0:12025:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9210. 00000400:00000010:2.0:1773941414.263155:0:12025:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0cfb480. 00000100:00000010:2.0:1773941414.263158:0:12025:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b90f26800. 00000100:00000200:2.0:1773941414.263170:0:12025:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:3.0:1773941414.263185:0:12017:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b84ed4a00. 00000100:00000010:3.0:1773941414.263313:0:12026:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5b850686c0. 00000020:00000010:3.0:1773941414.263319:0:12026:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068bc0. 00000020:00000010:3.0:1773941414.263321:0:12026:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84ed4e00. 00000020:00000010:3.0:1773941414.263323:0:12026:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b90c3ce00. 00000020:00000010:3.0:1773941414.263325:0:12026:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9de0. 00000040:00000010:3.0:1773941414.263327:0:12026:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba216be00. 00000020:00000010:3.0:1773941414.263329:0:12026:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d96000. 00000020:00000010:3.0:1773941414.263331:0:12026:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b88498a10. 00000020:00000010:3.0:1773941414.263333:0:12026:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba216a000. 80000000:00000010:3.0:1773941414.263335:0:12026:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc97e0. 40000000:00000010:3.0:1773941414.263336:0:12026:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b85068580. 00008000:00000010:3.0:1773941414.263338:0:12026:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b98a1c710. 00100000:00000010:3.0:1773941414.263340:0:12026:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b90d92000. 00040000:00000010:3.0:1773941414.263343:0:12026:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b884be600. 00040000:00000010:3.0:1773941414.263344:0:12026:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b90d96800. 00040000:00000010:3.0:1773941414.263346:0:12026:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba216bc00. 00000004:00000010:3.0:1773941414.263348:0:12026:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b90d97000. 00000004:00000010:3.0:1773941414.263351:0:12026:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5ba0b8c000. 00000004:00000010:3.0:1773941414.263353:0:12026:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b8907e800. 00002000:00000010:3.0:1773941414.263355:0:12026:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0b91800. 00000004:00000010:3.0:1773941414.263357:0:12026:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5ba0b92800. 00000100:00000010:3.0:1773941414.263359:0:12026:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b8907b000. 00000100:00000200:3.0:1773941414.263367:0:12026:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:1.0:1773941414.263382:0:12017:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5ba0f91200. 00000100:00000010:3.0:1773941414.263539:0:12027:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5b85068d80. 00000020:00000010:3.0:1773941414.263542:0:12027:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068740. 00000020:00000010:3.0:1773941414.263545:0:12027:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b99105000. 00000020:00000010:3.0:1773941414.263547:0:12027:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b99104800. 00000020:00000010:3.0:1773941414.263548:0:12027:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9240. 00000040:00000010:3.0:1773941414.263550:0:12027:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b98162600. 00000020:00000010:3.0:1773941414.263552:0:12027:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5ba21ce800. 00000020:00000010:3.0:1773941414.263554:0:12027:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b88499420. 00000020:00000010:3.0:1773941414.263556:0:12027:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b839e2600. 80000000:00000010:3.0:1773941414.263558:0:12027:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9300. 40000000:00000010:3.0:1773941414.263559:0:12027:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b85068c40. 00008000:00000010:3.0:1773941414.263565:0:12027:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b98a1fc08. 00100000:00000010:3.0:1773941414.263567:0:12027:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b9066a800. 00040000:00000010:3.0:1773941414.263569:0:12027:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b884bec00. 00040000:00000010:3.0:1773941414.263571:0:12027:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b9066d000. 00040000:00000010:3.0:1773941414.263573:0:12027:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b839e2a00. 00000004:00000010:3.0:1773941414.263575:0:12027:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b9066e800. 00000004:00000010:3.0:1773941414.263577:0:12027:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5ba0b88000. 00000004:00000010:3.0:1773941414.263579:0:12027:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b9066a000. 00002000:00000010:3.0:1773941414.263581:0:12027:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0b93c00. 00000004:00000010:3.0:1773941414.263582:0:12027:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5ba0b91000. 00000100:00000010:3.0:1773941414.263584:0:12027:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b90668800. 00000100:00000200:3.0:1773941414.263593:0:12027:0:(service.c:3086:ptlrpc_main()) service thread 2 (#3) started 00010000:00000010:1.0:1773941414.263602:0:12017:0:(ldlm_lockd.c:3466:ldlm_setup()) kmalloced '(blp)': 168 at ffff8b5b885de480. 00010000:00000010:3.0:1773941414.263764:0:12028:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at ffff8b5b85068d40. 00000020:00000010:3.0:1773941414.263769:0:12028:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b839e3000. 00000020:00000010:3.0:1773941414.263772:0:12028:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b839e3a00. 00000020:00000010:3.0:1773941414.263775:0:12028:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b9066b800. 00000020:00000010:3.0:1773941414.263779:0:12028:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b8849cb78. 00000020:00000010:3.0:1773941414.263783:0:12028:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b91e54c00. 80000000:00000010:3.0:1773941414.263786:0:12028:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9000. 40000000:00000010:3.0:1773941414.263788:0:12028:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b85068780. 00100000:00000010:3.0:1773941414.263790:0:12028:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b9066b000. 00040000:00000010:3.0:1773941414.263792:0:12028:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b884be900. 00040000:00000010:3.0:1773941414.263793:0:12028:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b91e55200. 00002000:00000010:3.0:1773941414.263795:0:12028:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0b92400. 00000020:00000010:3.0:1773941414.263797:0:12028:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b850680c0. 00010000:00000010:3.0:1773941414.263921:0:12029:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at ffff8b5b85068d00. 00000020:00000010:3.0:1773941414.263924:0:12029:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84d8ac00. 00000020:00000010:3.0:1773941414.263926:0:12029:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b84d8a800. 00000020:00000010:3.0:1773941414.263928:0:12029:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b9066e000. 00000020:00000010:3.0:1773941414.263930:0:12029:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b8849e4a0. 00000020:00000010:3.0:1773941414.263931:0:12029:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b84d8aa00. 80000000:00000010:3.0:1773941414.263932:0:12029:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9360. 40000000:00000010:3.0:1773941414.263934:0:12029:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b85068a80. 00100000:00000010:3.0:1773941414.263940:0:12029:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b9026d000. 00040000:00000010:3.0:1773941414.263942:0:12029:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b884beb00. 00040000:00000010:3.0:1773941414.263944:0:12029:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b84d8ba00. 00002000:00000010:3.0:1773941414.263945:0:12029:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0b91c00. 00000020:00000010:3.0:1773941414.263947:0:12029:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068280. 00000020:00000010:3.0:1773941414.264075:0:12030:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84d8ae00. 00000020:00000010:3.0:1773941414.264077:0:12030:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b84d8a600. 00000040:00000010:3.0:1773941414.264079:0:12030:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b84d8bc00. 00000020:00000010:3.0:1773941414.264081:0:12030:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b9026a000. 00000020:00000010:3.0:1773941414.264083:0:12030:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b8849eeb0. 00000020:00000010:3.0:1773941414.264085:0:12030:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b84d8a200. 80000000:00000010:3.0:1773941414.264087:0:12030:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9d20. 40000000:00000010:3.0:1773941414.264088:0:12030:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b85068380. 00100000:00000010:3.0:1773941414.264090:0:12030:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b9026e000. 00040000:00000010:3.0:1773941414.264091:0:12030:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b884be000. 00040000:00000010:3.0:1773941414.264093:0:12030:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b9026c800. 00040000:00000010:3.0:1773941414.264095:0:12030:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b84d8be00. 00000004:00000010:3.0:1773941414.264097:0:12030:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b9026f800. 00000004:00000010:3.0:1773941414.264099:0:12030:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5ba0b8a000. 00000004:00000010:3.0:1773941414.264109:0:12030:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b9026f000. 00002000:00000010:3.0:1773941414.264111:0:12030:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0b91400. 00000004:00000010:3.0:1773941414.264112:0:12030:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5ba0b92c00. 00000020:00000010:3.0:1773941414.264116:0:12030:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068fc0. 00000001:00000010:1.0:1773941414.264136:0:12017:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at ffff8b5b84cfb720. 00000001:00000010:1.0:1773941414.264146:0:12017:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at ffff8b5b84cfbea0. 00000020:00000010:1.0:1773941414.264164:0:12017:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at ffff8b5b88954000. 00000100:00000200:1.0:1773941414.264182:0:12017:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.143@tcp->12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941414.264189:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.264194:0:12017:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.143@tcp primary NID 192.168.204.143@tcp rc 0 00010000:00000010:1.0:1773941414.264200:0:12017:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at ffff8b5b84cfb4e0. 00010000:00000010:1.0:1773941414.264205:0:12017:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at ffff8b5ba0c29400. 00000001:00000010:1.0:1773941414.264209:0:12017:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at ffff8b5ba0f90000. 00000001:00000010:1.0:1773941414.264213:0:12017:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 16 at ffff8b5b84f3f310. 00000001:00000010:1.0:1773941414.264216:0:12017:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 304 at ffff8b5ba0f90600. 00000001:00000010:1.0:1773941414.264219:0:12017:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 304 at ffff8b5ba0f90200. 00000020:00000010:1.0:1773941414.264279:0:12017:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5ba0c2bc00. 00000020:00000010:1.0:1773941414.264282:0:12017:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at ffff8b5b84cfb960. 00000020:00000010:1.0:1773941414.264341:0:12017:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5ba0c28800. 00000020:00000010:1.0:1773941414.264348:0:12017:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at ffff8b5ba0f91600. 00000040:00000010:1.0:1773941414.264363:0:12017:0:(llog_obd.c:26:llog_new_ctxt()) kmalloced '(ctxt)': 168 at ffff8b5b885de3c0. 00000020:00000010:3.0:1773941414.264653:0:12017:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b903b6200. 00000040:00000010:3.0:1773941414.264658:0:12017:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b903b7200. 00000020:00000010:3.0:1773941414.264660:0:12017:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e6800. 00000020:00000010:3.0:1773941414.264663:0:12017:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b84e8e400. 00040000:00000010:3.0:1773941414.264667:0:12017:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5ba0c22a00. 00040000:00000010:3.0:1773941414.264670:0:12017:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b839c6e00. 00000020:00000010:3.0:1773941414.264673:0:12017:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b902a1a00. 00000020:00000010:3.0:1773941414.264677:0:12017:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8b5b90edb680. 00000020:00000010:3.0:1773941414.264679:0:12017:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8b5ba0c2f4b0. 00008000:00000010:3.0:1773941414.264681:0:12017:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8b5ba0b143e8. 00000020:00000010:3.0:1773941414.264688:0:12017:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b902a0800. 00000020:00000010:3.0:1773941414.264692:0:12017:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 104 at ffff8b5b90edb900. 00000020:00000010:3.0:1773941414.264719:0:12017:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5b885e6800. 00000400:00000200:3.0:1773941414.264731:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.264735:0:12017:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000010:3.0:1773941414.264745:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277480. 02000000:00000010:3.0:1773941414.264756:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0b90800. 00000020:00000010:3.0:1773941414.264785:0:12017:0:(obd_mount.c:561:lustre_start_mgc()) kfreed 'uuid': 40 at ffff8b5b82c59440. 00000020:00000010:3.0:1773941414.264790:0:12017:0:(obd_mount.c:562:lustre_start_mgc()) kfreed 'data': 192 at ffff8b5b85339e40. 00000020:00000010:3.0:1773941414.264794:0:12017:0:(obd_mount.c:563:lustre_start_mgc()) kfreed 'mgcname': 23 at ffff8b5b88570e20. 02000000:00000010:1.0:1773941414.264796:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0c29800. 00000080:00000010:3.0:1773941414.264799:0:12017:0:(llite_lib.c:1399:ll_fill_super()) kmalloced '(cfg)': 88 at ffff8b5ba0bc95a0. 00000400:00000200:1.0:1773941414.264805:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941414.264809:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034c08. 00000080:00000010:3.0:1773941414.264811:0:12017:0:(llite_lib.c:95:ll_init_sbi()) kmalloced '(sbi)': 1376 at ffff8b5b98ac1000. 00000400:00000010:1.0:1773941414.264816:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1a18. 00000100:00000200:1.0:1773941414.264822:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241140224, portal 25 00000400:00000010:1.0:1773941414.264827:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f15d8. 00000100:00000200:1.0:1773941414.264830:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241140224, offset 0 00000400:00000200:1.0:1773941414.264839:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941414.264847:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000010:3.0:1773941414.264851:0:12017:0:(cl_page.c:1380:cl_cache_init()) kmalloced '(cache)': 152 at ffff8b5b85339e40. 00000400:00000200:1.0:1773941414.264855:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000080:00000010:3.0:1773941414.264856:0:12017:0:(llite_lib.c:129:ll_init_sbi()) kmalloced '(sbi->ll_foreign_symlink_prefix)': 6 at ffff8b5b893196b0. 00000080:00000010:3.0:1773941414.264861:0:12017:0:(llite_lib.c:136:ll_init_sbi()) kmalloced '(sbi->ll_foreign_symlink_upcall)': 5 at ffff8b5b89319488. 00000400:00000200:1.0:1773941414.264863:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941414.264868:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941414.264874:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941414.264878:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941414.264886:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [74:74:256:4294967295] 192.168.204.143@tcp LPNI seq info [74:74:8:4294967295] 00000400:00000200:1.0:1773941414.264891:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.264906:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941414.264912:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941414.264917:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5ba0c22300. 00000800:00000200:1.0:1773941414.264925:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941414.264932:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941414.264937:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5ba0c22300 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:1.0:1773941414.264951:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941414.264956:0:2351:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b5b84cfb480. 00000100:00000010:1.0:1773941414.264967:0:2351:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b5b84cfba80. 00000020:00000010:3.0:1773941414.265185:0:12017:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5b90261c00. 00000020:00000010:3.0:1773941414.265189:0:12017:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 1440 at ffff8b5b98ac3000. 00000020:00000010:3.0:1773941414.265200:0:12017:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5b90262c00. 00000020:00000010:3.0:1773941414.265203:0:12017:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 576 at ffff8b5b90263c00. 00000020:00000010:3.0:1773941414.265211:0:12017:0:(obd_mount.c:67:lustre_process_log()) kmalloced '(bufs)': 104 at ffff8b5b9020a700. 00000020:00000010:3.0:1773941414.265214:0:12017:0:(obd_mount.c:76:lustre_process_log()) kmalloced '(lcfg)': 184 at ffff8b5b85028d80. 00000020:00000010:3.0:1773941414.265216:0:12017:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84d8a000. 00000020:00000010:3.0:1773941414.265219:0:12017:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b84d8b400. 00000040:00000010:3.0:1773941414.265222:0:12017:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b84d8b200. 00000020:00000010:3.0:1773941414.265224:0:12017:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b98ac5800. 00000020:00000010:3.0:1773941414.265226:0:12017:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b902a0800. 00040000:00000010:3.0:1773941414.265229:0:12017:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b884be400. 00040000:00000010:3.0:1773941414.265231:0:12017:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b902a1a00. 10000000:00000010:3.0:1773941414.265239:0:12017:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 287 at ffff8b5b839c6e00. 02000000:00000010:3.0:1773941414.265243:0:12017:0:(sec_config.c:546:sptlrpc_conf_get()) kmalloced '(conf)': 120 at ffff8b5b9020ab80. 00000100:00000010:3.0:1773941414.265247:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277100. 02000000:00000010:3.0:1773941414.265252:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b84e8e400. 00010000:00000010:3.0:1773941414.265266:0:12017:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962f00. 00010000:00000010:3.0:1773941414.265270:0:12017:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000010:3.0:1773941414.265273:0:12017:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff8b5b9020a800. 00000100:00000010:3.0:1773941414.265283:0:12017:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b85028480. 00000020:00000010:1.0:1773941414.265788:0:255:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 448 at ffff8b5ba0f90800. 00000800:00000200:2.0:1773941414.268381:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941414.268387:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5ba0c22300. 00000400:00000200:2.0:1773941414.268408:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.268416:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941414.268423:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f15d8 00000400:00000010:2.0:1773941414.268426:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f15d8. 00000100:00000200:2.0:1773941414.268436:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277480 x1860112241140224/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941424 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941414.269741:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.269746:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941414.269751:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.269755:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.269759:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.269763:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321602200 00000400:00000200:2.0:1773941414.269768:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.269774:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x735 [1] + 0 00000400:00000200:2.0:1773941414.269788:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.269792:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941414.269807:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277480 x1860112241140224/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941424 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941414.269861:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034c08. 00000400:00000200:1.0:1773941414.269868:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1a18 00000400:00000010:1.0:1773941414.269870:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1a18. 00000100:00000200:1.0:1773941414.269884:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277480 x1860112241140224/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941424 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941414.269895:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277480 x1860112241140224/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941424 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1773941414.269951:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0c29800. 02000000:00000010:1.0:1773941414.269955:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0b90800. 00000100:00000010:1.0:1773941414.269959:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277480. 02000000:00000010:2.0:1773941414.269979:0:12017:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b83f0a400. 00000400:00000200:2.0:1773941414.269986:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941414.269990:0:12017:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af93c8. 00000400:00000010:2.0:1773941414.269995:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34550. 00000100:00000200:2.0:1773941414.270001:0:12017:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241140352, portal 25 00000400:00000010:2.0:1773941414.270006:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f347f8. 00000100:00000200:2.0:1773941414.270010:0:12017:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241140352, offset 0 00000400:00000200:2.0:1773941414.270016:0:12017:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941414.270022:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.270027:0:12017:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941414.270030:0:12017:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941414.270033:0:12017:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941414.270036:0:12017:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941414.270038:0:12017:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941414.270042:0:12017:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [75:75:256:4294967295] 192.168.204.143@tcp LPNI seq info [75:75:8:4294967295] 00000400:00000200:2.0:1773941414.270045:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.270051:0:12017:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941414.270056:0:12017:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941414.270060:0:12017:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0800. 00000800:00000200:2.0:1773941414.270065:0:12017:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941414.270070:0:12017:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941414.270073:0:12017:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0800 type 1, nob 424 niov 1 nkiov 1 00000400:00000200:2.0:1773941414.270083:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:2.0:1773941414.270137:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941414.270141:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0800. 00000400:00000200:2.0:1773941414.270146:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.270150:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941414.270153:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f347f8 00000400:00000010:2.0:1773941414.270155:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f347f8. 00000100:00000200:2.0:1773941414.270168:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277100 x1860112241140352/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941414.271180:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.271184:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941414.271188:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.271190:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.271192:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.271195:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 344 into portal 25 MB=0x69bc321602280 00000400:00000200:2.0:1773941414.271198:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.271201:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 344/344 into md 0x73d [1] + 224 00000400:00000200:2.0:1773941414.271208:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.271210:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941414.271215:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277100 x1860112241140352/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941414.271252:0:12017:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af93c8. 00000400:00000200:3.0:1773941414.271258:0:12017:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34550 00000400:00000010:3.0:1773941414.271259:0:12017:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34550. 00000100:00000200:3.0:1773941414.271267:0:12017:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277100 x1860112241140352/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941414.271278:0:12017:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277100 x1860112241140352/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941414.271293:0:12017:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b85028480. 02000000:00000010:3.0:1773941414.271310:0:12017:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b83f0a400. 02000000:00000010:3.0:1773941414.271314:0:12017:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b84e8e400. 00000100:00000010:3.0:1773941414.271318:0:12017:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277100. 10000000:00000010:3.0:1773941414.271327:0:12017:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068e80. 00000020:00000010:3.0:1773941414.271331:0:12017:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b845c5400. 00000020:00000010:3.0:1773941414.271337:0:12017:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b845c5e00. 00000040:00000010:3.0:1773941414.271342:0:12017:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b845c4600. 00000020:00000010:3.0:1773941414.271346:0:12017:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b98ac7000. 00000020:00000010:3.0:1773941414.271350:0:12017:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b845c4200. 80000000:00000010:3.0:1773941414.271354:0:12017:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc93c0. 00040000:00000010:3.0:1773941414.271359:0:12017:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b845c4a00. 20000000:00000010:3.0:1773941414.271364:0:12017:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b84e8e400. 00000040:00000010:3.0:1773941414.271371:0:12017:0:(llog.c:42:llog_alloc_handle()) kmalloced '(loghandle)': 352 at ffff8b5b8809ca00. 00000100:00000010:3.0:1773941414.271376:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277100. 02000000:00000010:3.0:1773941414.271387:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b8809da00. 00000100:00000010:3.0:1773941414.271409:0:12017:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b85028480. 02000000:00000010:3.0:1773941414.271424:0:12017:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b8809c800. 00000400:00000200:3.0:1773941414.271428:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941414.271431:0:12017:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87aa8. 00000400:00000010:3.0:1773941414.271439:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6aaa0. 00000100:00000200:3.0:1773941414.271441:0:12017:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241140480, portal 25 00000400:00000010:3.0:1773941414.271444:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6ae58. 00000100:00000200:3.0:1773941414.271446:0:12017:0:(niobuf.c:94:ptl_send_buf()) Sending 512 bytes to portal 26, xid 1860112241140480, offset 0 00000400:00000200:3.0:1773941414.271450:0:12017:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941414.271454:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.271457:0:12017:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941414.271461:0:12017:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941414.271463:0:12017:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941414.271466:0:12017:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941414.271469:0:12017:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:3.0:1773941414.271472:0:12017:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [76:76:256:4294967295] 192.168.204.143@tcp LPNI seq info [76:76:8:4294967295] 00000400:00000200:3.0:1773941414.271476:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.271482:0:12017:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:3.0:1773941414.271487:0:12017:0:(socklnd_cb.c:1001:ksocknal_send()) sending 512 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:3.0:1773941414.271492:0:12017:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b884be200. 00000800:00000200:3.0:1773941414.271496:0:12017:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:3.0:1773941414.271500:0:12017:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:3.0:1773941414.271502:0:12017:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b884be200 type 1, nob 608 niov 1 nkiov 1 00000400:00000200:3.0:1773941414.271513:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:2.0:1773941414.271582:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941414.271586:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b884be200. 00000400:00000200:2.0:1773941414.271590:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.271595:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941414.271598:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6ae58 00000400:00000010:2.0:1773941414.271600:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6ae58. 00000100:00000200:2.0:1773941414.271608:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277100 x1860112241140480/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941430 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941414.272387:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.272400:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941414.272406:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.272410:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.272414:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.272417:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 272 into portal 25 MB=0x69bc321602300 00000400:00000200:2.0:1773941414.272421:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.272424:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 272/272 into md 0x74d [1] + 224 00000400:00000200:2.0:1773941414.272442:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.272445:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941414.272459:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277100 x1860112241140480/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941414.272496:0:12017:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87aa8. 00000400:00000200:3.0:1773941414.272500:0:12017:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6aaa0 00000400:00000010:3.0:1773941414.272501:0:12017:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6aaa0. 00000100:00000200:3.0:1773941414.272507:0:12017:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277100 x1860112241140480/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941414.272514:0:12017:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277100 x1860112241140480/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941414.272524:0:12017:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b85028480. 02000000:00000010:3.0:1773941414.272527:0:12017:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b8809c800. 02000000:00000010:3.0:1773941414.272529:0:12017:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b8809da00. 00000100:00000010:3.0:1773941414.272537:0:12017:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277100. 00000040:00000010:3.0:1773941414.272539:0:12017:0:(llog.c:72:llog_free_handle()) kfreed 'loghandle': 352 at ffff8b5b8809ca00. 00000020:00000010:3.0:1773941414.272543:0:12017:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b845c5e00. 00000040:00000010:3.0:1773941414.272545:0:12017:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b845c4600. 00000020:00000010:3.0:1773941414.272547:0:12017:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b98ac7000. 00000020:00000010:3.0:1773941414.272551:0:12017:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b845c4200. 80000000:00000010:3.0:1773941414.272553:0:12017:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc93c0. 00040000:00000010:3.0:1773941414.272556:0:12017:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b845c4a00. 20000000:00000010:3.0:1773941414.272559:0:12017:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b84e8e400. 00000020:00000010:3.0:1773941414.272561:0:12017:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b845c5400. 10000000:00000010:3.0:1773941414.272564:0:12017:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068e80. 10000000:00000010:3.0:1773941414.272580:0:12017:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 279 at ffff8b5b8809ca00. 10000000:00000010:3.0:1773941414.272586:0:12017:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 286 at ffff8b5b8809da00. 10000000:00000010:3.0:1773941414.272590:0:12017:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 285 at ffff8b5b8809c800. 00000100:00000010:3.0:1773941414.272596:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277100. 02000000:00000010:3.0:1773941414.272602:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b8809d400. 00010000:00000010:3.0:1773941414.272613:0:12017:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81484a00. 00000100:00000010:3.0:1773941414.272625:0:12017:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b85028480. 02000000:00000010:3.0:1773941414.272632:0:12017:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90261400. 00000400:00000200:3.0:1773941414.272635:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941414.272638:0:12017:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87aa8. 00000400:00000010:3.0:1773941414.272641:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6aaa0. 00000100:00000200:3.0:1773941414.272643:0:12017:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241140608, portal 25 00000400:00000010:3.0:1773941414.272646:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6acc0. 00000100:00000200:3.0:1773941414.272648:0:12017:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241140608, offset 0 00000400:00000200:3.0:1773941414.272652:0:12017:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941414.272655:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.272658:0:12017:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941414.272660:0:12017:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941414.272662:0:12017:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941414.272664:0:12017:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941414.272666:0:12017:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:3.0:1773941414.272668:0:12017:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [77:77:256:4294967295] 192.168.204.143@tcp LPNI seq info [77:77:8:4294967295] 00000400:00000200:3.0:1773941414.272676:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.272681:0:12017:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:3.0:1773941414.272684:0:12017:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:3.0:1773941414.272688:0:12017:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b884be800. 00000800:00000200:3.0:1773941414.272691:0:12017:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:3.0:1773941414.272695:0:12017:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:3.0:1773941414.272697:0:12017:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b884be800 type 1, nob 424 niov 1 nkiov 1 00000400:00000200:3.0:1773941414.272727:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:2.0:1773941414.272773:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941414.272777:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b884be800. 00000400:00000200:2.0:1773941414.272781:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.272786:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941414.272789:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6acc0 00000400:00000010:2.0:1773941414.272791:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6acc0. 00000100:00000200:2.0:1773941414.272797:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277100 x1860112241140608/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941414.273667:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.273672:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941414.273677:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.273680:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.273683:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.273686:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 344 into portal 25 MB=0x69bc321602380 00000400:00000200:2.0:1773941414.273690:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.273694:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 344/344 into md 0x75d [1] + 224 00000400:00000200:2.0:1773941414.273731:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.273734:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941414.273739:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277100 x1860112241140608/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941414.273765:0:12017:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87aa8. 00000400:00000200:3.0:1773941414.273771:0:12017:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6aaa0 00000400:00000010:3.0:1773941414.273773:0:12017:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6aaa0. 00000100:00000200:3.0:1773941414.273780:0:12017:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277100 x1860112241140608/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941414.273791:0:12017:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277100 x1860112241140608/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941414.273806:0:12017:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b85028480. 02000000:00000010:3.0:1773941414.273818:0:12017:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90261400. 02000000:00000010:3.0:1773941414.273822:0:12017:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b8809d400. 00000100:00000010:3.0:1773941414.273825:0:12017:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277100. 10000000:00000010:3.0:1773941414.273830:0:12017:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068e80. 00000020:00000010:3.0:1773941414.273834:0:12017:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b8809d400. 00000020:00000010:3.0:1773941414.273837:0:12017:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b8809c200. 00000040:00000010:3.0:1773941414.273840:0:12017:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b8809c600. 00000020:00000010:3.0:1773941414.273844:0:12017:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b98ac7000. 00000020:00000010:3.0:1773941414.273848:0:12017:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b8809ce00. 80000000:00000010:3.0:1773941414.273851:0:12017:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc93c0. 00040000:00000010:3.0:1773941414.273855:0:12017:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b84e8e400. 20000000:00000010:3.0:1773941414.273866:0:12017:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b845c5400. 00000040:00000010:3.0:1773941414.273875:0:12017:0:(llog.c:42:llog_alloc_handle()) kmalloced '(loghandle)': 352 at ffff8b5b845c4a00. 00000100:00000010:3.0:1773941414.273880:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277100. 02000000:00000010:3.0:1773941414.273891:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b845c4200. 00000100:00000010:3.0:1773941414.273899:0:12017:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b85028480. 02000000:00000010:3.0:1773941414.273907:0:12017:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b845c4600. 00000400:00000200:3.0:1773941414.273912:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941414.273915:0:12017:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87aa8. 00000400:00000010:3.0:1773941414.273919:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6aaa0. 00000100:00000200:3.0:1773941414.273922:0:12017:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241140736, portal 25 00000400:00000010:3.0:1773941414.273925:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6add0. 00000100:00000200:3.0:1773941414.273928:0:12017:0:(niobuf.c:94:ptl_send_buf()) Sending 512 bytes to portal 26, xid 1860112241140736, offset 0 00000400:00000200:3.0:1773941414.273933:0:12017:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941414.273938:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.273942:0:12017:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941414.273945:0:12017:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941414.273948:0:12017:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941414.273951:0:12017:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941414.273953:0:12017:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:3.0:1773941414.273957:0:12017:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [78:78:256:4294967295] 192.168.204.143@tcp LPNI seq info [78:78:8:4294967295] 00000400:00000200:3.0:1773941414.273961:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.273968:0:12017:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:3.0:1773941414.273973:0:12017:0:(socklnd_cb.c:1001:ksocknal_send()) sending 512 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:3.0:1773941414.273976:0:12017:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b884be300. 00000800:00000200:3.0:1773941414.273981:0:12017:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:3.0:1773941414.273986:0:12017:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:3.0:1773941414.273989:0:12017:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b884be300 type 1, nob 608 niov 1 nkiov 1 00000400:00000200:3.0:1773941414.274003:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:2.0:1773941414.274055:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941414.274059:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b884be300. 00000400:00000200:2.0:1773941414.274063:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.274067:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941414.274070:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6add0 00000400:00000010:2.0:1773941414.274073:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6add0. 00000100:00000200:2.0:1773941414.274087:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277100 x1860112241140736/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941430 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941414.275087:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.275091:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941414.275097:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.275099:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.275102:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.275105:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 272 into portal 25 MB=0x69bc321602400 00000400:00000200:2.0:1773941414.275109:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.275113:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 272/272 into md 0x76d [1] + 224 00000400:00000200:2.0:1773941414.275122:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.275126:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941414.275132:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277100 x1860112241140736/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941414.275180:0:12017:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87aa8. 00000400:00000200:3.0:1773941414.275185:0:12017:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6aaa0 00000400:00000010:3.0:1773941414.275187:0:12017:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6aaa0. 00000100:00000200:3.0:1773941414.275193:0:12017:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277100 x1860112241140736/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941414.275204:0:12017:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277100 x1860112241140736/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941414.275222:0:12017:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b85028480. 02000000:00000010:3.0:1773941414.275228:0:12017:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b845c4600. 02000000:00000010:3.0:1773941414.275231:0:12017:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b845c4200. 00000100:00000010:3.0:1773941414.275233:0:12017:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277100. 00000040:00000010:3.0:1773941414.275240:0:12017:0:(llog.c:412:llog_init_handle()) kmalloced '(llh)': 8192 at ffff8b5b90d44000. 00000100:00000010:3.0:1773941414.275245:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277100. 02000000:00000010:3.0:1773941414.275250:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b845c4200. 00000100:00000010:3.0:1773941414.275255:0:12017:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b85028480. 02000000:00000010:3.0:1773941414.275272:0:12017:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at ffff8b5ba0ce4000. 00000400:00000200:3.0:1773941414.275276:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941414.275279:0:12017:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87aa8. 00000400:00000010:3.0:1773941414.275283:0:12017:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at ffff8b5b85028900. 00000100:00000200:3.0:1773941414.275286:0:12017:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112241140864, portal 25 00000400:00000010:3.0:1773941414.275289:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6aaa0. 00000100:00000200:3.0:1773941414.275291:0:12017:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112241140864, offset 0 00000400:00000200:3.0:1773941414.275296:0:12017:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941414.275299:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.275303:0:12017:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941414.275306:0:12017:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941414.275308:0:12017:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941414.275311:0:12017:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941414.275313:0:12017:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:3.0:1773941414.275316:0:12017:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [79:79:256:4294967295] 192.168.204.143@tcp LPNI seq info [79:79:8:4294967295] 00000400:00000200:3.0:1773941414.275320:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.275327:0:12017:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:3.0:1773941414.275331:0:12017:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:3.0:1773941414.275335:0:12017:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b884bed00. 00000800:00000200:3.0:1773941414.275338:0:12017:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:3.0:1773941414.275343:0:12017:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:3.0:1773941414.275346:0:12017:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b884bed00 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:3.0:1773941414.275364:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:2.0:1773941414.275414:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941414.275417:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b884bed00. 00000400:00000200:2.0:1773941414.275421:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.275425:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941414.275428:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6aaa0 00000400:00000010:2.0:1773941414.275430:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6aaa0. 00000100:00000200:2.0:1773941414.275435:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277100 x1860112241140864/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941430 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941414.276316:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.276321:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941414.276325:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.276327:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.276330:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.276333:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 8416 into portal 25 MB=0x69bc321602480 00000400:00000200:2.0:1773941414.276336:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.276339:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 8416/8416 into md 0x77d [4] + 224 00000400:00000200:2.0:1773941414.276355:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.276359:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941414.276365:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277100 x1860112241140864/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941414.276417:0:12017:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87aa8. 00000400:00000200:3.0:1773941414.276422:0:12017:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85028900 00000400:00000010:3.0:1773941414.276424:0:12017:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at ffff8b5b85028900. 00000100:00000200:3.0:1773941414.276431:0:12017:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277100 x1860112241140864/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941414.276441:0:12017:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277100 x1860112241140864/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941414.276458:0:12017:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b85028480. 02000000:00000010:3.0:1773941414.276466:0:12017:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at ffff8b5ba0ce4000. 02000000:00000010:3.0:1773941414.276479:0:12017:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b845c4200. 00000100:00000010:3.0:1773941414.276481:0:12017:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277100. 00000040:00000010:3.0:1773941414.276487:0:12017:0:(llog.c:945:llog_process_or_fork()) kmalloced '(lpi)': 96 at ffff8b5ba0bc9a20. 00000020:00000010:3.0:1773941414.276737:0:12035:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b845c4200. 00000020:00000010:3.0:1773941414.276742:0:12035:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b845c5e00. 00000040:00000010:3.0:1773941414.276747:0:12035:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b845c4600. 00000020:00000010:3.0:1773941414.276751:0:12035:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b98ac7800. 00000020:00000010:3.0:1773941414.276757:0:12035:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b88498f18. 00000020:00000010:3.0:1773941414.276763:0:12035:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b91df5e00. 80000000:00000010:3.0:1773941414.276767:0:12035:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9f60. 40000000:00000010:3.0:1773941414.276772:0:12035:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b85068500. 00100000:00000010:3.0:1773941414.276777:0:12035:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b98ac4800. 00040000:00000010:3.0:1773941414.276782:0:12035:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b884bee00. 00040000:00000010:3.0:1773941414.276787:0:12035:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b98ac2000. 00040000:00000010:3.0:1773941414.276791:0:12035:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b91df5600. 00000004:00000010:3.0:1773941414.276796:0:12035:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b98ac6800. 00000004:00000010:3.0:1773941414.276802:0:12035:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5ba0b8f000. 00000004:00000010:3.0:1773941414.276807:0:12035:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b98ac5000. 00002000:00000010:3.0:1773941414.276813:0:12035:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b90261400. 00000004:00000010:3.0:1773941414.276818:0:12035:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5b90263800. 00000020:00000010:3.0:1773941414.276823:0:12035:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068300. 00000040:00000010:3.0:1773941414.276830:0:12035:0:(llog.c:570:llog_process_thread()) kmalloced '(buf)': 8192 at ffff8b5b90d46000. 00000100:00000010:3.0:1773941414.276841:0:12035:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277100. 02000000:00000010:3.0:1773941414.276848:0:12035:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b91df5a00. 00000100:00000010:3.0:1773941414.276856:0:12035:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b85028900. 02000000:00000010:3.0:1773941414.276870:0:12035:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at ffff8b5ba0ce4000. 00000400:00000200:3.0:1773941414.276875:0:12035:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941414.276878:0:12035:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87aa8. 00000400:00000010:3.0:1773941414.276881:0:12035:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at ffff8b5b85028c00. 00000100:00000200:3.0:1773941414.276885:0:12035:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112241140992, portal 25 00000400:00000010:3.0:1773941414.276890:0:12035:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a6e8. 00000100:00000200:3.0:1773941414.276892:0:12035:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112241140992, offset 0 00000400:00000200:3.0:1773941414.276897:0:12035:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941414.276901:0:12035:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.276905:0:12035:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941414.276909:0:12035:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941414.276912:0:12035:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941414.276915:0:12035:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941414.276918:0:12035:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:3.0:1773941414.276921:0:12035:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [80:80:256:4294967295] 192.168.204.143@tcp LPNI seq info [80:80:8:4294967295] 00000400:00000200:3.0:1773941414.276926:0:12035:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.276933:0:12035:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:3.0:1773941414.276938:0:12035:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:3.0:1773941414.276942:0:12035:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b884bed00. 00000800:00000200:3.0:1773941414.276946:0:12035:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:3.0:1773941414.276951:0:12035:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:3.0:1773941414.276953:0:12035:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b884bed00 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:3.0:1773941414.276962:0:12035:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941414.277012:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941414.277016:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b884bed00. 00000400:00000200:3.0:1773941414.277019:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941414.277025:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941414.277028:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a6e8 00000400:00000010:3.0:1773941414.277037:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a6e8. 00000100:00000200:3.0:1773941414.277044:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277100 x1860112241140992/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941430 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941414.277883:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.277887:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941414.277891:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.277892:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.277895:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.277897:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 8472 into portal 25 MB=0x69bc321602500 00000400:00000200:3.0:1773941414.277900:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.277905:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 8472/8472 into md 0x78d [4] + 224 00000400:00000200:3.0:1773941414.277914:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941414.277917:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941414.277971:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277100 x1860112241140992/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941414.278012:0:12035:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87aa8. 00000400:00000200:1.0:1773941414.278021:0:12035:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85028c00 00000400:00000010:1.0:1773941414.278024:0:12035:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at ffff8b5b85028c00. 00000100:00000200:1.0:1773941414.278031:0:12035:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277100 x1860112241140992/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941414.278041:0:12035:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277100 x1860112241140992/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941414.278056:0:12035:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b85028900. 02000000:00000010:1.0:1773941414.278063:0:12035:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at ffff8b5ba0ce4000. 02000000:00000010:1.0:1773941414.278072:0:12035:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b91df5a00. 00000100:00000010:1.0:1773941414.278075:0:12035:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277100. 00000020:00000010:1.0:1773941414.278088:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff8b5ba0c22800. 00000020:00000010:1.0:1773941414.278092:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff8b5ba0c22800. 00000020:00000010:1.0:1773941414.278099:0:12035:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at ffff8b5b82c59900. 00000020:00000010:1.0:1773941414.278104:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 128 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.278114:0:12035:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at ffff8b5b85513370. 00000020:00000010:1.0:1773941414.278121:0:12035:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5b88951000. 00000020:00000010:1.0:1773941414.278135:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 128 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.278137:0:12035:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at ffff8b5b82c59900. 00000020:00000010:1.0:1773941414.278142:0:12035:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at ffff8b5b82c59900. 00000020:00000010:1.0:1773941414.278146:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 160 at ffff8b5b885de180. 00000001:00000010:1.0:1773941414.278179:0:12035:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at ffff8b5ba0c22800. 00000001:00000010:1.0:1773941414.278184:0:12035:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff8b5b88570040. 00000001:00000010:1.0:1773941414.278192:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5ba0f90c00. 00000001:00000010:1.0:1773941414.278194:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5ba0f91e00. 00000001:00000010:1.0:1773941414.278196:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5ba0f91000. 00000001:00000010:1.0:1773941414.278198:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5ba0f91800. 00000020:00000010:1.0:1773941414.278201:0:12035:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0f91a00. 00000020:00000010:1.0:1773941414.278205:0:12035:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.278208:0:12035:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8b5ba0c2f640. 00008000:00000010:1.0:1773941414.278211:0:12035:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5ba0b141c8. 00000020:00000010:1.0:1773941414.278215:0:12035:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0f90e00. 00000020:00000010:1.0:1773941414.278218:0:12035:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b91ecae00. 00000020:00000010:1.0:1773941414.278221:0:12035:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5b84cfb6c0. 00008000:00000010:1.0:1773941414.278225:0:12035:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d26350. 00020000:00000010:1.0:1773941414.278230:0:12035:0:(lov_dev.c:599:lov_device_alloc()) kmalloced '(ld)': 632 at ffff8b5ba0ce6000. 00000020:00000010:1.0:1773941414.278246:0:12035:0:(lu_tgt_pool.c:52:lu_tgt_pool_init()) kmalloced '(op->op_array)': 8 at ffff8b5b890ef778. 00020000:00000010:1.0:1773941414.278386:0:12035:0:(lov_dev.c:616:lov_device_alloc()) kmalloced '(ld->ld_md_tgts)': 4096 at ffff8b5b85e5f000. 00020000:00000010:1.0:1773941414.278405:0:12035:0:(lov_dev.c:623:lov_device_alloc()) kmalloced '(ld->ld_lov->lov_mdc_tgts)': 4096 at ffff8b5b85e5a000. 00000020:00000010:1.0:1773941414.278432:0:12035:0:(lu_object.c:1154:lu_site_init()) kmalloced '(s->ls_bkts)': 18432 at ffff8b5b90dd0000. 00000020:00000010:1.0:1773941414.278441:0:12035:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5ba0ce6c00. 00000020:00000010:1.0:1773941414.278444:0:12035:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 192 at ffff8b5b885de000. 00000020:00000010:1.0:1773941414.278453:0:12035:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b91ecae00. 00000020:00000010:1.0:1773941414.278456:0:12035:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5b84cfb6c0. 00008000:00000010:1.0:1773941414.278460:0:12035:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d26350. 00000020:00000010:1.0:1773941414.278463:0:12035:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0f90e00. 00000020:00000010:1.0:1773941414.278467:0:12035:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.278471:0:12035:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8b5ba0c2f640. 00008000:00000010:1.0:1773941414.278473:0:12035:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8b5ba0b141c8. 00000020:00000010:1.0:1773941414.278476:0:12035:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0f91a00. 00000020:00000010:1.0:1773941414.278480:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 160 at ffff8b5b885de180. 00000020:00000010:1.0:1773941414.278483:0:12035:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at ffff8b5b82c59900. 00000020:00000010:1.0:1773941414.278487:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff8b5ba0c22500. 00000020:00000010:1.0:1773941414.278489:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff8b5ba0c22500. 00000020:00000010:1.0:1773941414.278493:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff8b5ba0c22500. 00000020:00000010:1.0:1773941414.278499:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff8b5ba0c22500. 00000020:00000010:1.0:1773941414.278503:0:12035:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at ffff8b5b82c59900. 00000020:00000010:1.0:1773941414.278505:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 128 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.278511:0:12035:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at ffff8b5b85510000. 00000020:00000010:1.0:1773941414.278514:0:12035:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5b88950800. 00000020:00000010:1.0:1773941414.278530:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 128 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.278537:0:12035:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at ffff8b5b82c59900. 00000020:00000010:1.0:1773941414.278542:0:12035:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at ffff8b5b82c59900. 00000020:00000010:1.0:1773941414.278547:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 160 at ffff8b5b885de180. 00000001:00000010:1.0:1773941414.278566:0:12035:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at ffff8b5ba0c22500. 00000001:00000010:1.0:1773941414.278572:0:12035:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff8b5b88570ee0. 00000001:00000010:1.0:1773941414.278575:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5b91ecae00. 00000001:00000010:1.0:1773941414.278579:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5b91ecbc00. 00000001:00000010:1.0:1773941414.278581:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5b91eca600. 00000001:00000010:1.0:1773941414.278583:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5b91ecb800. 00000020:00000010:1.0:1773941414.278586:0:12035:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0f91a00. 00000020:00000010:1.0:1773941414.278590:0:12035:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.278593:0:12035:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8b5ba0c2f640. 00020000:00000010:1.0:1773941414.278597:0:12035:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5ba0b9f840. 00008000:00000010:1.0:1773941414.278599:0:12035:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5ba0b141c8. 00000020:00000010:1.0:1773941414.278602:0:12035:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0f90e00. 00000020:00000010:1.0:1773941414.278615:0:12035:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0c91c00. 00000040:00000010:1.0:1773941414.278619:0:12035:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0c90400. 00000020:00000010:1.0:1773941414.278623:0:12035:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b88950000. 00000020:00000010:1.0:1773941414.278627:0:12035:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0c91400. 00040000:00000010:1.0:1773941414.278631:0:12035:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5ba0c22c00. 00040000:00000010:1.0:1773941414.278635:0:12035:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0c90a00. 00800000:00000010:1.0:1773941414.278640:0:12035:0:(lmv_obd.c:1219:lmv_device_alloc()) kmalloced '(lu)': 64 at ffff8b5b82c59280. 00000020:00000010:1.0:1773941414.278962:0:12035:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5ba0ce4400. 00000020:00000010:1.0:1773941414.278966:0:12035:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 512 at ffff8b5ba0c91000. 80000000:00000010:1.0:1773941414.278984:0:12035:0:(fld_cache.c:47:fld_cache_init()) kmalloced '(cache)': 168 at ffff8b5b885de240. 00800000:00000010:1.0:1773941414.279029:0:12035:0:(lmv_obd.c:1279:lmv_device_alloc()) kmalloced '(pat)': 280 at ffff8b5ba0c90800. 00800000:00000010:1.0:1773941414.279032:0:12035:0:(lmv_obd.c:1288:lmv_device_alloc()) kmalloced '(pat)': 280 at ffff8b5ba0c90200. 00000020:00000010:1.0:1773941414.279034:0:12035:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0c91c00. 00000040:00000010:1.0:1773941414.279037:0:12035:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0c90400. 00000020:00000010:1.0:1773941414.279040:0:12035:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b88950000. 00000020:00000010:1.0:1773941414.279044:0:12035:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0c91400. 00040000:00000010:1.0:1773941414.279047:0:12035:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5ba0c22c00. 00040000:00000010:1.0:1773941414.279050:0:12035:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0c90a00. 00000020:00000010:1.0:1773941414.279053:0:12035:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0f90e00. 00000020:00000010:1.0:1773941414.279056:0:12035:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.279059:0:12035:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8b5ba0c2f640. 00020000:00000010:1.0:1773941414.279062:0:12035:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5ba0b9f840. 00008000:00000010:1.0:1773941414.279065:0:12035:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8b5ba0b141c8. 00000020:00000010:1.0:1773941414.279068:0:12035:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0f91a00. 00000020:00000010:1.0:1773941414.279071:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 160 at ffff8b5b885de180. 00000020:00000010:1.0:1773941414.279074:0:12035:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at ffff8b5b82c59900. 00000020:00000010:1.0:1773941414.279078:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff8b5ba0c22c00. 00000020:00000010:1.0:1773941414.279080:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff8b5ba0c22c00. 00000020:00000010:1.0:1773941414.279084:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff8b5ba0c22c00. 00000020:00000010:1.0:1773941414.279086:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff8b5ba0c22c00. 00000020:00000010:1.0:1773941414.279091:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 96 at ffff8b5b84cfb6c0. 00000020:00000010:1.0:1773941414.279094:0:12035:0:(obd_config.c:1186:class_add_profile()) kmalloced '(lprof)': 48 at ffff8b5b82c59900. 00000020:00000010:1.0:1773941414.279097:0:12035:0:(obd_config.c:1192:class_add_profile()) kmalloced '(lprof->lp_profile)': 14 at ffff8b5b84f3fb90. 00000020:00000010:1.0:1773941414.279099:0:12035:0:(obd_config.c:1198:class_add_profile()) kmalloced '(lprof->lp_dt)': 14 at ffff8b5b84f3f4d0. 00000020:00000010:1.0:1773941414.279102:0:12035:0:(obd_config.c:1205:class_add_profile()) kmalloced '(lprof->lp_md)': 14 at ffff8b5b84f3fee0. 00000020:00000010:1.0:1773941414.279105:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 96 at ffff8b5b84cfb6c0. 00000020:00000010:1.0:1773941414.279110:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff8b5ba0c22c00. 00000020:00000010:1.0:1773941414.279112:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff8b5ba0c22c00. 00000020:00000010:1.0:1773941414.279120:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff8b5ba0c22c00. 00000020:00000010:1.0:1773941414.279123:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff8b5ba0c22c00. 00000020:00000010:1.0:1773941414.279127:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at ffff8b5b82c59600. 00000100:00000010:1.0:1773941414.279133:0:12035:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at ffff8b5ba0ce5000. 00000400:00000200:1.0:1773941414.279143:0:12035:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:1.0:1773941414.279148:0:12035:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.143@tcp 00000400:00000200:1.0:1773941414.279152:0:12035:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.279156:0:12035:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.143@tcp NID flags 0x100001: -17 00000400:00000200:1.0:1773941414.279161:0:12035:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.279167:0:12035:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1773941414.279171:0:12035:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp: 0. pending discovery 00000400:00000200:1.0:1773941414.279174:0:12035:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.143@tcp 00000100:00000010:1.0:1773941414.279177:0:12035:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at ffff8b5ba0ce5000. 00000020:00000010:1.0:1773941414.279181:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 64 at ffff8b5b82c59600. 00000020:00000010:1.0:1773941414.279188:0:12035:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at ffff8b5b82c59600. 00000020:00000010:1.0:1773941414.279193:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at ffff8b5b885de180. 00000020:00000010:1.0:1773941414.279200:0:12035:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at ffff8b5b855119b8. 00000020:00000010:1.0:1773941414.279204:0:12035:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5b88950000. 00000020:00000010:1.0:1773941414.279213:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at ffff8b5b885de180. 00000020:00000010:1.0:1773941414.279216:0:12035:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at ffff8b5b82c59600. 00000020:00000010:1.0:1773941414.279220:0:12035:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at ffff8b5b82c59600. 00000020:00000010:1.0:1773941414.279224:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at ffff8b5b885de180. 00000001:00000010:1.0:1773941414.279256:0:12035:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at ffff8b5ba0c22c00. 00000001:00000010:1.0:1773941414.279263:0:12035:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff8b5b88570800. 00000001:00000010:1.0:1773941414.279267:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5ba0c90a00. 00000001:00000010:1.0:1773941414.279271:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5ba0c91400. 00000001:00000010:1.0:1773941414.279274:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5ba0c90400. 00000001:00000010:1.0:1773941414.279278:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5ba0c91c00. 00000020:00000010:1.0:1773941414.279283:0:12035:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0c90e00. 00000020:00000010:1.0:1773941414.279288:0:12035:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.279293:0:12035:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8b5ba0c2f640. 00020000:00000010:1.0:1773941414.279296:0:12035:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5ba0b9f840. 00008000:00000010:1.0:1773941414.279299:0:12035:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5ba0b141c8. 00000020:00000010:1.0:1773941414.279302:0:12035:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0c91600. 00000020:00000010:1.0:1773941414.279305:0:12035:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0c91200. 00000020:00000010:1.0:1773941414.279308:0:12035:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5b84cfb6c0. 00020000:00000010:1.0:1773941414.279316:0:12035:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a10df00. 00008000:00000010:1.0:1773941414.279318:0:12035:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d26350. 00000002:00000010:1.0:1773941414.279322:0:12035:0:(mdc_dev.c:1726:mdc_device_alloc()) kmalloced '(osc)': 104 at ffff8b5b90edb580. 00010000:00000010:1.0:1773941414.279332:0:12035:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at ffff8b5b85e5c000. 00010000:00000010:1.0:1773941414.279339:0:12035:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at ffff8b5b85e5e000. 00010000:00000010:1.0:1773941414.279346:0:12035:0:(ldlm_lib.c:522:client_obd_setup()) kmalloced '(cli->cl_mod_tag_bitmap)': 64 at ffff8b5b82c59340. 00000020:00000010:1.0:1773941414.279350:0:12035:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at ffff8b5b88953800. 00000100:00000200:1.0:1773941414.279359:0:12035:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.143@tcp->12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941414.279362:0:12035:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.279365:0:12035:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.143@tcp primary NID 192.168.204.143@tcp rc 0 00010000:00000010:1.0:1773941414.279370:0:12035:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at ffff8b5b84cfb8a0. 00010000:00000010:1.0:1773941414.279374:0:12035:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at ffff8b5ba0c9e800. 00000001:00000010:1.0:1773941414.279378:0:12035:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at ffff8b5ba0c90000. 00000001:00000010:1.0:1773941414.279381:0:12035:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 256 at ffff8b5ba0c22100. 00000001:00000010:1.0:1773941414.279427:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41cd5000. 00000001:00000010:1.0:1773941414.279470:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41cdf000. 00000001:00000010:1.0:1773941414.279508:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41ce9000. 00000001:00000010:1.0:1773941414.279560:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41cf3000. 00000001:00000010:1.0:1773941414.279611:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41cfd000. 00000001:00000010:1.0:1773941414.279653:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41d07000. 00000001:00000010:1.0:1773941414.279695:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41d11000. 00000001:00000010:1.0:1773941414.279785:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41d1b000. 00000001:00000010:1.0:1773941414.279844:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41d25000. 00000001:00000010:1.0:1773941414.279901:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41d2f000. 00000001:00000010:1.0:1773941414.279963:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41d39000. 00000001:00000010:1.0:1773941414.280019:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41d4d000. 00000001:00000010:1.0:1773941414.280073:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41d57000. 00000001:00000010:1.0:1773941414.280130:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41d61000. 00000001:00000010:1.0:1773941414.280191:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41d6b000. 00000001:00000010:1.0:1773941414.280253:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41d75000. 00000001:00000010:1.0:1773941414.280312:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41d7f000. 00000001:00000010:1.0:1773941414.280378:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41d89000. 00000001:00000010:1.0:1773941414.280441:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41d93000. 00000001:00000010:1.0:1773941414.280491:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41d9d000. 00000001:00000010:1.0:1773941414.280542:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41dad000. 00000001:00000010:1.0:1773941414.280599:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41dbd000. 00000001:00000010:1.0:1773941414.280655:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41dc7000. 00000001:00000010:1.0:1773941414.280728:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41dd1000. 00000001:00000010:1.0:1773941414.280786:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41ddb000. 00000001:00000010:1.0:1773941414.280843:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41de5000. 00000001:00000010:1.0:1773941414.280902:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41df5000. 00000001:00000010:1.0:1773941414.280963:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41e1d000. 00000001:00000010:1.0:1773941414.281020:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41e27000. 00000001:00000010:1.0:1773941414.281076:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41e31000. 00000001:00000010:1.0:1773941414.281130:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41e3b000. 00000001:00000010:1.0:1773941414.281187:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffa10c41e45000. 00000020:00000010:1.0:1773941414.281317:0:12035:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5ba0c9e000. 00000020:00000010:1.0:1773941414.281322:0:12035:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at ffff8b5b880f9720. 00000020:00000010:1.0:1773941414.281433:0:12035:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5ba0c9f400. 00000020:00000010:1.0:1773941414.281438:0:12035:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at ffff8b5ba0c91800. 00000020:00000010:1.0:1773941414.281690:0:12035:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5ba0c9c000. 00000020:00000010:1.0:1773941414.281694:0:12035:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 512 at ffff8b5ba0c91e00. 00000020:00000010:1.0:1773941414.281751:0:12035:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5ba0c9f000. 00000020:00000010:1.0:1773941414.281755:0:12035:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at ffff8b5b85e5d000. 00000040:00000010:1.0:1773941414.281779:0:12035:0:(llog_obd.c:26:llog_new_ctxt()) kmalloced '(ctxt)': 168 at ffff8b5b885de540. 00000002:00000010:1.0:1773941414.281787:0:12035:0:(mdc_changelog.c:816:mdc_changelog_cdev_init()) kmalloced '(entry)': 1304 at ffff8b5b88951800. 00000020:00000010:1.0:1773941414.281974:0:12035:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0c91200. 00000020:00000010:1.0:1773941414.281979:0:12035:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5b84cfb6c0. 00020000:00000010:1.0:1773941414.281984:0:12035:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a10df00. 00008000:00000010:1.0:1773941414.281988:0:12035:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d26350. 00000020:00000010:1.0:1773941414.281992:0:12035:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0c91600. 00000020:00000010:1.0:1773941414.281996:0:12035:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.282006:0:12035:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8b5ba0c2f640. 00020000:00000010:1.0:1773941414.282010:0:12035:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5ba0b9f840. 00008000:00000010:1.0:1773941414.282014:0:12035:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8b5ba0b141c8. 00000020:00000010:1.0:1773941414.282017:0:12035:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0c90e00. 00000020:00000010:1.0:1773941414.282022:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at ffff8b5b885de180. 00000020:00000010:1.0:1773941414.282025:0:12035:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at ffff8b5b82c59600. 00000020:00000010:1.0:1773941414.282032:0:12035:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at ffff8b5b83dcd6c0. 00000020:00000010:1.0:1773941414.282036:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 152 at ffff8b5b885de180. 00000020:00000010:1.0:1773941414.282043:0:12035:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0c90e00. 00000020:00000010:1.0:1773941414.282046:0:12035:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0c91600. 00000040:00000010:1.0:1773941414.282050:0:12035:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0c91200. 00000020:00000010:1.0:1773941414.282053:0:12035:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b88956000. 00000020:00000010:1.0:1773941414.282057:0:12035:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0c90c00. 00040000:00000010:1.0:1773941414.282061:0:12035:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5ba0c22a00. 00040000:00000010:1.0:1773941414.282064:0:12035:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0c91a00. 00800000:00000010:1.0:1773941414.282075:0:12035:0:(lmv_obd.c:433:lmv_add_target()) kmalloced '(tgt)': 368 at ffff8b5ba0f91a00. 00020000:00000010:1.0:1773941414.282080:0:12035:0:(lu_tgt_descs.c:371:ltd_add_tgt()) kmalloced '(ltd->ltd_tgt_idx[index / (((1UL) << 12) / sizeof(void *))])': 4096 at ffff8b5b90f31000. 00000020:00000010:1.0:1773941414.282083:0:12035:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0c91600. 00000040:00000010:1.0:1773941414.282087:0:12035:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0c91200. 00000020:00000010:1.0:1773941414.282090:0:12035:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b88956000. 00000020:00000010:1.0:1773941414.282094:0:12035:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0c90c00. 00040000:00000010:1.0:1773941414.282096:0:12035:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5ba0c22a00. 00040000:00000010:1.0:1773941414.282100:0:12035:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0c91a00. 00000020:00000010:1.0:1773941414.282102:0:12035:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0c90e00. 00000020:00000010:1.0:1773941414.282107:0:12035:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0f90e00. 00000020:00000010:1.0:1773941414.282110:0:12035:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0c90e00. 00000020:00000010:1.0:1773941414.282113:0:12035:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5b880f99c0. 00020000:00000010:1.0:1773941414.282116:0:12035:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a10df00. 00008000:00000010:1.0:1773941414.282120:0:12035:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d26350. 00000020:00000010:1.0:1773941414.282128:0:12035:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0c90e00. 00000020:00000010:1.0:1773941414.282131:0:12035:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5b880f99c0. 00020000:00000010:1.0:1773941414.282135:0:12035:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a10df00. 00008000:00000010:1.0:1773941414.282145:0:12035:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d26350. 00000020:00000010:1.0:1773941414.282148:0:12035:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0f90e00. 00000020:00000010:1.0:1773941414.282151:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 152 at ffff8b5b885de180. 00000020:00000010:1.0:1773941414.282154:0:12035:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at ffff8b5b83dcd6c0. 00000020:00000010:1.0:1773941414.282158:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff8b5ba0c22a00. 00000020:00000010:1.0:1773941414.282161:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff8b5ba0c22a00. 00000020:00000010:1.0:1773941414.282166:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff8b5ba0c22a00. 00000020:00000010:1.0:1773941414.282168:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff8b5ba0c22a00. 00000020:00000010:1.0:1773941414.282174:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff8b5ba0c22a00. 00000020:00000010:1.0:1773941414.282176:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff8b5ba0c22a00. 00000020:00000010:1.0:1773941414.282179:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at ffff8b5b83dcd6c0. 00000100:00000010:1.0:1773941414.282183:0:12035:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at ffff8b5ba0c9ec00. 00000400:00000200:1.0:1773941414.282186:0:12035:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:1.0:1773941414.282191:0:12035:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.143@tcp 00000400:00000200:1.0:1773941414.282196:0:12035:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.282200:0:12035:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.143@tcp NID flags 0x100001: -17 00000400:00000200:1.0:1773941414.282204:0:12035:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.282209:0:12035:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1773941414.282212:0:12035:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp: 0. pending discovery 00000400:00000200:1.0:1773941414.282214:0:12035:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.143@tcp 00000100:00000010:1.0:1773941414.282216:0:12035:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at ffff8b5ba0c9ec00. 00000020:00000010:1.0:1773941414.282219:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 64 at ffff8b5b83dcd6c0. 00000020:00000010:1.0:1773941414.282222:0:12035:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at ffff8b5b83dcd6c0. 00000020:00000010:1.0:1773941414.282226:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at ffff8b5b885de180. 00000020:00000010:1.0:1773941414.282241:0:12035:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at ffff8b5ba07399b8. 00000020:00000010:1.0:1773941414.282247:0:12035:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5b88956000. 00000020:00000010:1.0:1773941414.282256:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at ffff8b5b885de180. 00000020:00000010:1.0:1773941414.282259:0:12035:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at ffff8b5b83dcd6c0. 00000020:00000010:1.0:1773941414.282264:0:12035:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at ffff8b5b83dcd6c0. 00000020:00000010:1.0:1773941414.282268:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at ffff8b5b885de180. 00000001:00000010:1.0:1773941414.282292:0:12035:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at ffff8b5ba0c22a00. 00000001:00000010:1.0:1773941414.282297:0:12035:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff8b5b88570bc0. 00000001:00000010:1.0:1773941414.282307:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5ba0c90e00. 00000001:00000010:1.0:1773941414.282311:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5ba0c91a00. 00000001:00000010:1.0:1773941414.282313:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5ba0c90c00. 00000001:00000010:1.0:1773941414.282316:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5ba0c91200. 00000020:00000010:1.0:1773941414.282320:0:12035:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0c91600. 00000020:00000010:1.0:1773941414.282323:0:12035:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.282327:0:12035:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8b5ba0c2f640. 00020000:00000010:1.0:1773941414.282330:0:12035:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5ba0b9f840. 00008000:00000010:1.0:1773941414.282332:0:12035:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5ba0b141c8. 00000020:00000010:1.0:1773941414.282336:0:12035:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0f90e00. 00000020:00000010:1.0:1773941414.282347:0:12035:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941414.282350:0:12035:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5b880f99c0. 00020000:00000010:1.0:1773941414.282352:0:12035:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a10df00. 00008000:00000010:1.0:1773941414.282355:0:12035:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d26350. 00000008:00000010:1.0:1773941414.282359:0:12035:0:(osc_dev.c:192:osc_device_alloc()) kmalloced '(osc)': 104 at ffff8b5b90edb300. 00010000:00000010:1.0:1773941414.282370:0:12035:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at ffff8b5b90f36000. 00010000:00000010:1.0:1773941414.282377:0:12035:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at ffff8b5b90f33000. 00000020:00000010:1.0:1773941414.282384:0:12035:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at ffff8b5b88957800. 00000100:00000200:1.0:1773941414.282411:0:12035:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.143@tcp->12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941414.282416:0:12035:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.282420:0:12035:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.143@tcp primary NID 192.168.204.143@tcp rc 0 00010000:00000010:1.0:1773941414.282427:0:12035:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at ffff8b5b880f9d80. 00010000:00000010:1.0:1773941414.282431:0:12035:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at ffff8b5ba0736000. 00000001:00000010:1.0:1773941414.282435:0:12035:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at ffff8b5b850b7600. 00000001:00000010:1.0:1773941414.282439:0:12035:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 128 at ffff8b5b90edbb80. 00000001:00000010:1.0:1773941414.282452:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5b850bc000. 00000001:00000010:1.0:1773941414.282459:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5b850be000. 00000001:00000010:1.0:1773941414.282465:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5b850ba000. 00000001:00000010:1.0:1773941414.282470:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5b850b8000. 00000001:00000010:1.0:1773941414.282480:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5b885c6000. 00000001:00000010:1.0:1773941414.282484:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5b885c2000. 00000001:00000010:1.0:1773941414.282488:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5b885c0000. 00000001:00000010:1.0:1773941414.282496:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5b885c4000. 00000001:00000010:1.0:1773941414.282507:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5b885ca000. 00000001:00000010:1.0:1773941414.282511:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5b885c8000. 00000001:00000010:1.0:1773941414.282515:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5b885cc000. 00000001:00000010:1.0:1773941414.282519:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5b885ce000. 00000001:00000010:1.0:1773941414.282528:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5ba0cb6000. 00000001:00000010:1.0:1773941414.282533:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5ba0cb2000. 00000001:00000010:1.0:1773941414.282537:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5ba0cb0000. 00000001:00000010:1.0:1773941414.282542:0:12035:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff8b5ba0cb4000. 00000020:00000010:1.0:1773941414.282604:0:12035:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5ba0737400. 00000020:00000010:1.0:1773941414.282607:0:12035:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at ffff8b5b880f9960. 00000020:00000010:1.0:1773941414.282657:0:12035:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5ba0734000. 00000020:00000010:1.0:1773941414.282661:0:12035:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at ffff8b5b850b7200. 00000020:00000010:1.0:1773941414.282945:0:12035:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5ba0737000. 00000020:00000010:1.0:1773941414.282949:0:12035:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at ffff8b5b90f37000. 00000020:00000010:1.0:1773941414.282970:0:12035:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941414.282973:0:12035:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5b880f99c0. 00020000:00000010:1.0:1773941414.282976:0:12035:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a10df00. 00008000:00000010:1.0:1773941414.282978:0:12035:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d26350. 00000020:00000010:1.0:1773941414.282981:0:12035:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0f90e00. 00000020:00000010:1.0:1773941414.282985:0:12035:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.282988:0:12035:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8b5ba0c2f640. 00020000:00000010:1.0:1773941414.282991:0:12035:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5ba0b9f840. 00008000:00000010:1.0:1773941414.282993:0:12035:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8b5ba0b141c8. 00000020:00000010:1.0:1773941414.282997:0:12035:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0c91600. 00000020:00000010:1.0:1773941414.283000:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at ffff8b5b885de180. 00000020:00000010:1.0:1773941414.283002:0:12035:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at ffff8b5b83dcd6c0. 00000020:00000010:1.0:1773941414.283007:0:12035:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at ffff8b5b850b0f00. 00000020:00000010:1.0:1773941414.283011:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 120 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.283015:0:12035:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941414.283018:0:12035:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941414.283025:0:12035:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5b880f99c0. 00000008:00000010:1.0:1773941414.283029:0:12035:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5b90d8ce18. 00020000:00000010:1.0:1773941414.283032:0:12035:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a10df00. 00008000:00000010:1.0:1773941414.283034:0:12035:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d26350. 00020000:00000010:1.0:1773941414.283043:0:12035:0:(lov_obd.c:492:lov_add_target()) kmalloced '(tgt)': 368 at ffff8b5b850b6600. 00020000:00000010:1.0:1773941414.283047:0:12035:0:(lu_tgt_descs.c:371:ltd_add_tgt()) kmalloced '(ltd->ltd_tgt_idx[index / (((1UL) << 12) / sizeof(void *))])': 4096 at ffff8b5b90f34000. 00020000:00000010:1.0:1773941414.283052:0:12035:0:(lov_dev.c:357:lov_expand_targets()) kmalloced '(newd)': 512 at ffff8b5b850b7e00. 00000020:00000010:1.0:1773941414.283054:0:12035:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941414.283056:0:12035:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5b880f99c0. 00000008:00000010:1.0:1773941414.283058:0:12035:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5b90d8ce18. 00020000:00000010:1.0:1773941414.283061:0:12035:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a10df00. 00008000:00000010:1.0:1773941414.283064:0:12035:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d26350. 00000020:00000010:1.0:1773941414.283065:0:12035:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941414.283067:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 120 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.283069:0:12035:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at ffff8b5b850b0f00. 00000020:00000010:1.0:1773941414.283074:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff8b5b9024b100. 00000020:00000010:1.0:1773941414.283076:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff8b5b9024b100. 00000020:00000010:1.0:1773941414.283080:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff8b5b9024b100. 00000020:00000010:1.0:1773941414.283081:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff8b5b9024b100. 00000020:00000010:1.0:1773941414.283085:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 56 at ffff8b5b850b0f00. 00000020:00000010:1.0:1773941414.283086:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 56 at ffff8b5b850b0f00. 00000100:00000010:1.0:1773941414.283094:0:12035:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60a80. 02000000:00000010:1.0:1773941414.283105:0:12035:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b6e00. 00000100:00000010:1.0:1773941414.283117:0:12035:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b885de180. 02000000:00000010:1.0:1773941414.283144:0:12035:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at ffff8b5b889d0000. 00000400:00000200:1.0:1773941414.283150:0:12035:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941414.283155:0:12035:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034c08. 00000400:00000010:1.0:1773941414.283161:0:12035:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at ffff8b5b885dee40. 00000100:00000200:1.0:1773941414.283167:0:12035:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112241141120, portal 25 00000400:00000010:1.0:1773941414.283173:0:12035:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1a18. 00000100:00000200:1.0:1773941414.283176:0:12035:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112241141120, offset 0 00000400:00000200:1.0:1773941414.283181:0:12035:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941414.283192:0:12035:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.283196:0:12035:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941414.283198:0:12035:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941414.283202:0:12035:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941414.283205:0:12035:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941414.283208:0:12035:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941414.283211:0:12035:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [81:81:256:4294967295] 192.168.204.143@tcp LPNI seq info [81:81:8:4294967295] 00000400:00000200:1.0:1773941414.283214:0:12035:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.283224:0:12035:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941414.283228:0:12035:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941414.283231:0:12035:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b100. 00000800:00000200:1.0:1773941414.283235:0:12035:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941414.283241:0:12035:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941414.283244:0:12035:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b100 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:1.0:1773941414.283261:0:12035:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941414.283334:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941414.283339:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b100. 00000400:00000200:3.0:1773941414.283345:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941414.283352:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941414.283357:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1a18 00000400:00000010:3.0:1773941414.283360:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1a18. 00000100:00000200:3.0:1773941414.283370:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60a80 x1860112241141120/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941430 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941414.285059:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.285066:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941414.285072:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.285076:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.285080:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.285085:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 8472 into portal 25 MB=0x69bc321602580 00000400:00000200:2.0:1773941414.285090:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.285096:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 8472/8472 into md 0x79d [4] + 224 00000400:00000200:2.0:1773941414.285112:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.285116:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941414.285125:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60a80 x1860112241141120/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941414.285179:0:12035:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034c08. 00000400:00000200:1.0:1773941414.285184:0:12035:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b885dee40 00000400:00000010:1.0:1773941414.285186:0:12035:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at ffff8b5b885dee40. 00000100:00000200:1.0:1773941414.285193:0:12035:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60a80 x1860112241141120/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941414.285204:0:12035:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60a80 x1860112241141120/t0(0) o502->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941414.285234:0:12035:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b885de180. 02000000:00000010:1.0:1773941414.285246:0:12035:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at ffff8b5b889d0000. 02000000:00000010:1.0:1773941414.285261:0:12035:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b6e00. 00000100:00000010:1.0:1773941414.285265:0:12035:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60a80. 00000020:00000010:1.0:1773941414.285274:0:12035:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff8b5b9024b800. 00000020:00000010:1.0:1773941414.285278:0:12035:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff8b5b9024b800. 00000040:00000010:1.0:1773941414.285282:0:12035:0:(llog.c:891:llog_process_thread()) kvfree 'buf': 8192 at ffff8b5b90d46000. 00000020:00000010:1.0:1773941414.285289:0:12035:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068300. 00000020:00000010:1.0:1773941414.285307:0:12035:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b845c5e00. 00000040:00000010:1.0:1773941414.285311:0:12035:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b845c4600. 00000020:00000010:1.0:1773941414.285314:0:12035:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b98ac7800. 00000020:00000010:1.0:1773941414.285319:0:12035:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b88498f18. 00000020:00000010:1.0:1773941414.285324:0:12035:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b91df5e00. 80000000:00000010:1.0:1773941414.285328:0:12035:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9f60. 40000000:00000010:1.0:1773941414.285333:0:12035:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b85068500. 00100000:00000010:1.0:1773941414.285338:0:12035:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b98ac4800. 00040000:00000010:1.0:1773941414.285342:0:12035:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b884bee00. 00040000:00000010:1.0:1773941414.285348:0:12035:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b98ac2000. 00040000:00000010:1.0:1773941414.285355:0:12035:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b91df5600. 00000004:00000010:1.0:1773941414.285360:0:12035:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b98ac6800. 00000004:00000010:1.0:1773941414.285366:0:12035:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5ba0b8f000. 00000004:00000010:1.0:1773941414.285372:0:12035:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b98ac5000. 00002000:00000010:1.0:1773941414.285377:0:12035:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5b90261400. 00000004:00000010:1.0:1773941414.285381:0:12035:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5b90263800. 00000020:00000010:1.0:1773941414.285400:0:12035:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b845c4200. 00000040:00000010:3.0:1773941414.285435:0:12017:0:(llog.c:986:llog_process_or_fork()) kfreed 'lpi': 96 at ffff8b5ba0bc9a20. 00000040:00000010:3.0:1773941414.285442:0:12017:0:(llog.c:70:llog_free_handle()) kvfree 'loghandle->lgh_hdr': 8192 at ffff8b5b90d44000. 00000040:00000010:3.0:1773941414.285445:0:12017:0:(llog.c:72:llog_free_handle()) kfreed 'loghandle': 352 at ffff8b5b845c4a00. 00000020:00000010:3.0:1773941414.285449:0:12017:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b8809c200. 00000040:00000010:3.0:1773941414.285453:0:12017:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b8809c600. 00000020:00000010:3.0:1773941414.285455:0:12017:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b98ac7000. 00000020:00000010:3.0:1773941414.285458:0:12017:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b8809ce00. 80000000:00000010:3.0:1773941414.285461:0:12017:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc93c0. 00040000:00000010:3.0:1773941414.285469:0:12017:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b84e8e400. 20000000:00000010:3.0:1773941414.285472:0:12017:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b845c5400. 00000020:00000010:3.0:1773941414.285476:0:12017:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b8809d400. 10000000:00000010:3.0:1773941414.285479:0:12017:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068e80. 00000100:00000010:3.0:1773941414.285506:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275880. 02000000:00000010:3.0:1773941414.285514:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b91df4200. 00010000:00000010:3.0:1773941414.285523:0:12017:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962000. 00010000:00000010:3.0:1773941414.285529:0:12017:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81484600. 00000100:00000010:3.0:1773941414.285544:0:12017:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b88ea73c0. 02000000:00000010:3.0:1773941414.285557:0:12017:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260800. 00000400:00000200:3.0:1773941414.285562:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941414.285567:0:12017:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87c60. 00000400:00000010:3.0:1773941414.285573:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a6e8. 00000100:00000200:3.0:1773941414.285578:0:12017:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241141248, portal 25 00000400:00000010:3.0:1773941414.285581:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a660. 00000100:00000200:3.0:1773941414.285584:0:12017:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241141248, offset 0 00000400:00000200:3.0:1773941414.285590:0:12017:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941414.285594:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.285598:0:12017:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941414.285601:0:12017:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941414.285604:0:12017:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941414.285606:0:12017:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941414.285608:0:12017:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:3.0:1773941414.285611:0:12017:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [82:82:256:4294967295] 192.168.204.143@tcp LPNI seq info [82:82:8:4294967295] 00000400:00000200:3.0:1773941414.285613:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.285617:0:12017:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:3.0:1773941414.285621:0:12017:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:3.0:1773941414.285625:0:12017:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b884be300. 00000800:00000200:3.0:1773941414.285628:0:12017:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:3.0:1773941414.285633:0:12017:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:3.0:1773941414.285635:0:12017:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b884be300 type 1, nob 424 niov 1 nkiov 1 00000400:00000200:3.0:1773941414.285650:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941414.285747:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941414.285751:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b884be300. 00000400:00000200:3.0:1773941414.285755:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941414.285759:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941414.285831:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a660 00000400:00000010:3.0:1773941414.285834:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a660. 00000100:00000200:3.0:1773941414.285842:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275880 x1860112241141248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941414.286620:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.286624:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941414.286627:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.286631:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.286635:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.286638:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 344 into portal 25 MB=0x69bc321602600 00000400:00000200:3.0:1773941414.286642:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.286647:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 344/344 into md 0x7a5 [1] + 224 00000400:00000200:3.0:1773941414.286656:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941414.286660:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941414.286667:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275880 x1860112241141248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941414.286694:0:12017:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87c60. 00000400:00000200:3.0:1773941414.286697:0:12017:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a6e8 00000400:00000010:3.0:1773941414.286698:0:12017:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a6e8. 00000100:00000200:3.0:1773941414.286702:0:12017:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275880 x1860112241141248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941414.286728:0:12017:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275880 x1860112241141248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941414.286741:0:12017:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b88ea73c0. 02000000:00000010:3.0:1773941414.286754:0:12017:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260800. 02000000:00000010:3.0:1773941414.286762:0:12017:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b91df4200. 00000100:00000010:3.0:1773941414.286764:0:12017:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275880. 10000000:00000010:3.0:1773941414.286770:0:12017:0:(mgc_request.c:1545:mgc_process_recover_log()) kmalloced '(pages)': 2048 at ffff8b5b98ac7000. 00000100:00000010:3.0:1773941414.287208:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275880. 02000000:00000010:3.0:1773941414.287215:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b91df4200. 00000100:00000010:3.0:1773941414.287220:0:12017:0:(client.c:169:ptlrpc_new_bulk()) kmalloced '(desc)': 984 at ffff8b5b90260800. 00000100:00000010:3.0:1773941414.287224:0:12017:0:(client.c:174:ptlrpc_new_bulk()) kmalloced '(desc->bd_vec)': 4096 at ffff8b5ba0b8b000. 00000100:00000010:3.0:1773941414.287234:0:12017:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b88ea73c0. 00000400:00000200:3.0:1773941414.287243:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941414.287244:0:12017:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87c60. 00000400:00000010:3.0:1773941414.287249:0:12017:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 4216 at ffff8b5b90d44000. 00000100:00000200:3.0:1773941414.287253:0:12017:0:(niobuf.c:453:ptlrpc_register_bulk()) Setup 1 bulk put-sink buffers: 256 pages 1048576 bytes, mbits x0x69bc321602680-0x69bc321602680, portal 33 02000000:00000010:3.0:1773941414.287257:0:12017:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b91df5000. 00000400:00000200:3.0:1773941414.287259:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941414.287261:0:12017:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87210. 00000400:00000010:3.0:1773941414.287264:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a6e8. 00000100:00000200:3.0:1773941414.287266:0:12017:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241141376, portal 25 00000400:00000010:3.0:1773941414.287268:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a660. 00000100:00000200:3.0:1773941414.287270:0:12017:0:(niobuf.c:94:ptl_send_buf()) Sending 304 bytes to portal 26, xid 1860112241141376, offset 0 00000400:00000200:3.0:1773941414.287274:0:12017:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941414.287277:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.287280:0:12017:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941414.287282:0:12017:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941414.287284:0:12017:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941414.287286:0:12017:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941414.287288:0:12017:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:3.0:1773941414.287291:0:12017:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [83:83:256:4294967295] 192.168.204.143@tcp LPNI seq info [83:83:8:4294967295] 00000400:00000200:3.0:1773941414.287293:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.287299:0:12017:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:3.0:1773941414.287302:0:12017:0:(socklnd_cb.c:1001:ksocknal_send()) sending 304 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:3.0:1773941414.287304:0:12017:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b884be300. 00000800:00000200:3.0:1773941414.287311:0:12017:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:3.0:1773941414.287316:0:12017:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:3.0:1773941414.287318:0:12017:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b884be300 type 1, nob 400 niov 1 nkiov 1 00000400:00000200:3.0:1773941414.287327:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941414.287373:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941414.287375:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b884be300. 00000400:00000200:3.0:1773941414.287377:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941414.287381:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941414.287383:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a660 00000400:00000010:3.0:1773941414.287384:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a660. 00000100:00000200:3.0:1773941414.287400:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275880 x1860112241141376/t0(0) o256->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941430 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941414.288293:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.288299:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941414.288306:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.288309:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.288313:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.288318:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 88 into portal 33 MB=0x69bc321602680 00000400:00000200:3.0:1773941414.288322:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.288327:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 21 from 12345-192.168.204.143@tcp of length 88/88 into md 0x7b5 [256] + 0 00000400:00000010:3.0:1773941414.288330:0:2347:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87c60. 00000400:00000200:3.0:1773941414.288334:0:2347:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md ffff8b5b90d44000 00000400:00000200:3.0:1773941414.288343:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941414.288348:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941414.288353:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d44000 00000400:00000010:3.0:1773941414.288355:0:2347:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 4216 at ffff8b5b90d44000. 00000100:00000200:3.0:1773941414.288361:0:2347:0:(events.c:192:client_bulk_callback()) event type 2, status 0, req ffff8b5b90275880 desc ffff8b5b90260800 mbits 1860112241141376 00000400:00000200:3.0:1773941414.288374:0:2347:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:3.0:1773941414.288379:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.288385:0:2347:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source Specified: 192.168.204.43@tcp to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941414.288402:0:2347:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [84:84:256:4294967295] 192.168.204.143@tcp LPNI seq info [84:84:8:4294967295] 00000400:00000200:3.0:1773941414.288413:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.288423:0:2347:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:192.168.204.43@tcp) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) 192.168.204.143@tcp : ACK try# 0 00000800:00000200:3.0:1773941414.288429:0:2347:0:(socklnd_cb.c:1001:ksocknal_send()) sending 0 bytes in 256 frags to 12345-192.168.204.143@tcp 00000800:00000010:3.0:1773941414.288434:0:2347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 4328 at ffff8b5b90d44000. 00000800:00000200:3.0:1773941414.288440:0:2347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:3.0:1773941414.288447:0:2347:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:3.0:1773941414.288450:0:2347:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b90d44000 type 0, nob 96 niov 1 nkiov 0 00000800:00000200:3.0:1773941414.288493:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941414.288496:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 4328 at ffff8b5b90d44000. 00000400:00000200:3.0:1773941414.288500:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941414.288504:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: ACK: OK 00000400:00000200:3.0:1773941414.289108:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.289112:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941414.289115:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.289116:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.289119:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.289123:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 240 into portal 25 MB=0x69bc321602680 00000400:00000200:3.0:1773941414.289126:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.289129:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 240/240 into md 0x7bd [1] + 224 00000400:00000200:3.0:1773941414.289137:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941414.289140:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941414.289147:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275880 x1860112241141376/t0(0) o256->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941414.289191:0:12017:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87210. 00000400:00000200:1.0:1773941414.289195:0:12017:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a6e8 00000400:00000010:1.0:1773941414.289196:0:12017:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a6e8. 00000100:00000200:1.0:1773941414.289202:0:12017:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275880 x1860112241141376/t0(0) o256->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941414.289209:0:12017:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275880 x1860112241141376/t0(0) o256->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941414.289225:0:12017:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b88ea73c0. 10000000:00000010:1.0:1773941414.289233:0:12017:0:(mgc_request.c:1297:mgc_apply_recover_logs()) kmalloced '(buf)': 4096 at ffff8b5b90f35000. 00000100:00000200:1.0:1773941414.289241:0:12017:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.204.143@tcp with 192.168.204.143@tcp 00000100:00000200:1.0:1773941414.289243:0:12017:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.204.143@tcp' 00000400:00000200:1.0:1773941414.289245:0:12017:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:1.0:1773941414.289248:0:12017:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.143@tcp 00000400:00000200:1.0:1773941414.289250:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.289253:0:12017:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.143@tcp NID flags 0x100001: -17 00000400:00000200:1.0:1773941414.289255:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.289263:0:12017:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1773941414.289265:0:12017:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp: 0. pending discovery 00000400:00000200:1.0:1773941414.289267:0:12017:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.143@tcp 10000000:00000010:1.0:1773941414.289271:0:12017:0:(mgc_request.c:1479:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.289278:0:12017:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941414.289281:0:12017:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941414.289283:0:12017:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5b880f99c0. 00000008:00000010:1.0:1773941414.289288:0:12017:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5b90d8ce18. 00020000:00000010:1.0:1773941414.289292:0:12017:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a10df00. 00008000:00000010:1.0:1773941414.289296:0:12017:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d26350. 00000020:00000010:1.0:1773941414.289327:0:12017:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941414.289330:0:12017:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5b880f99c0. 00000008:00000010:1.0:1773941414.289333:0:12017:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5b90d8ce18. 00020000:00000010:1.0:1773941414.289337:0:12017:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a10df00. 00008000:00000010:1.0:1773941414.289340:0:12017:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d26350. 00000020:00000010:1.0:1773941414.289342:0:12017:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b6e00. 10000000:00000010:1.0:1773941414.289344:0:12017:0:(mgc_request.c:1495:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at ffff8b5b90edb800. 00000100:00000200:1.0:1773941414.289348:0:12017:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.204.143@tcp with 192.168.204.143@tcp 00000100:00000200:1.0:1773941414.289349:0:12017:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.204.143@tcp' 00000400:00000200:1.0:1773941414.289350:0:12017:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:1.0:1773941414.289352:0:12017:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.143@tcp 00000400:00000200:1.0:1773941414.289354:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.289355:0:12017:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.143@tcp NID flags 0x100001: -17 00000400:00000200:1.0:1773941414.289357:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.289359:0:12017:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1773941414.289360:0:12017:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp: 0. pending discovery 00000400:00000200:1.0:1773941414.289361:0:12017:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.143@tcp 10000000:00000010:1.0:1773941414.289364:0:12017:0:(mgc_request.c:1479:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at ffff8b5b90edb800. 00000020:00000010:1.0:1773941414.289367:0:12017:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941414.289369:0:12017:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941414.289370:0:12017:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5b880f99c0. 00000008:00000010:1.0:1773941414.289372:0:12017:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5b90d8ce18. 00020000:00000010:1.0:1773941414.289373:0:12017:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a10df00. 00008000:00000010:1.0:1773941414.289378:0:12017:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d26350. 00000020:00000010:1.0:1773941414.289407:0:12017:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941414.289409:0:12017:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5b880f99c0. 00000008:00000010:1.0:1773941414.289411:0:12017:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5b90d8ce18. 00020000:00000010:1.0:1773941414.289412:0:12017:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a10df00. 00008000:00000010:1.0:1773941414.289414:0:12017:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d26350. 00000020:00000010:1.0:1773941414.289416:0:12017:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b6e00. 10000000:00000010:1.0:1773941414.289417:0:12017:0:(mgc_request.c:1495:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at ffff8b5b90edb800. 10000000:00000010:1.0:1773941414.289418:0:12017:0:(mgc_request.c:1507:mgc_apply_recover_logs()) kfreed 'buf': 4096 at ffff8b5b90f35000. 02000000:00000010:1.0:1773941414.289422:0:12017:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b91df5000. 00000100:00000010:1.0:1773941414.289430:0:12017:0:(client.c:333:ptlrpc_free_bulk()) kvfree 'desc->bd_vec': 4096 at ffff8b5ba0b8b000. 00000100:00000010:1.0:1773941414.289432:0:12017:0:(client.c:334:ptlrpc_free_bulk()) kfreed 'desc': 984 at ffff8b5b90260800. 02000000:00000010:1.0:1773941414.289434:0:12017:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b91df4200. 00000100:00000010:1.0:1773941414.289436:0:12017:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275880. 10000000:00000010:1.0:1773941414.289898:0:12017:0:(mgc_request.c:1663:mgc_process_recover_log()) kvfree 'pages': 2048 at ffff8b5b98ac7000. 00000100:00000010:1.0:1773941414.289916:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60a80. 02000000:00000010:1.0:1773941414.289923:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b6e00. 00010000:00000010:1.0:1773941414.289932:0:12017:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec800. 00010000:00000010:1.0:1773941414.289936:0:12017:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138fa00. 00000020:00000010:1.0:1773941414.289940:0:12017:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff8b5b90edb800. 00000100:00000010:1.0:1773941414.289950:0:12017:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b885de180. 02000000:00000010:1.0:1773941414.289958:0:12017:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735c00. 00000400:00000200:1.0:1773941414.289961:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941414.289964:0:12017:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034c08. 00000400:00000010:1.0:1773941414.289968:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f12a8. 00000100:00000200:1.0:1773941414.289970:0:12017:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241141504, portal 25 00000400:00000010:1.0:1773941414.289974:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1110. 00000100:00000200:1.0:1773941414.289976:0:12017:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241141504, offset 0 00000400:00000200:1.0:1773941414.289981:0:12017:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941414.289983:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.289986:0:12017:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941414.289988:0:12017:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941414.290003:0:12017:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941414.290005:0:12017:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941414.290006:0:12017:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941414.290008:0:12017:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [85:85:256:4294967295] 192.168.204.143@tcp LPNI seq info [85:85:8:4294967295] 00000400:00000200:1.0:1773941414.290011:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.290017:0:12017:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941414.290019:0:12017:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941414.290023:0:12017:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b800. 00000800:00000200:1.0:1773941414.290026:0:12017:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941414.290029:0:12017:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941414.290038:0:12017:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b800 type 1, nob 424 niov 1 nkiov 1 00000400:00000200:1.0:1773941414.290046:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941414.290089:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941414.290092:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b800. 00000400:00000200:3.0:1773941414.290095:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941414.290098:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941414.290100:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1110 00000400:00000010:3.0:1773941414.290101:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1110. 00000100:00000200:3.0:1773941414.290105:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60a80 x1860112241141504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941414.291087:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.291093:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941414.291100:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.291103:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.291107:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.291111:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 344 into portal 25 MB=0x69bc321602700 00000400:00000200:2.0:1773941414.291116:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.291120:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 344/344 into md 0x7cd [1] + 224 00000400:00000200:2.0:1773941414.291130:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.291134:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941414.291143:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60a80 x1860112241141504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941414.291185:0:12017:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034c08. 00000400:00000200:1.0:1773941414.291190:0:12017:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f12a8 00000400:00000010:1.0:1773941414.291192:0:12017:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f12a8. 00000100:00000200:1.0:1773941414.291198:0:12017:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60a80 x1860112241141504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941414.291208:0:12017:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60a80 x1860112241141504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941414.291223:0:12017:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b885de180. 02000000:00000010:1.0:1773941414.291236:0:12017:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735c00. 02000000:00000010:1.0:1773941414.291240:0:12017:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b6e00. 00000100:00000010:1.0:1773941414.291243:0:12017:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60a80. 10000000:00000010:1.0:1773941414.291249:0:12017:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0f00. 00000020:00000010:1.0:1773941414.291253:0:12017:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941414.291257:0:12017:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b7800. 00000040:00000010:1.0:1773941414.291262:0:12017:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6c00. 00000020:00000010:1.0:1773941414.291266:0:12017:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b88954800. 00000020:00000010:1.0:1773941414.291271:0:12017:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7c00. 80000000:00000010:1.0:1773941414.291276:0:12017:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f99c0. 00040000:00000010:1.0:1773941414.291287:0:12017:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6400. 20000000:00000010:1.0:1773941414.291292:0:12017:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b7400. 00000040:00000010:1.0:1773941414.291299:0:12017:0:(llog.c:42:llog_alloc_handle()) kmalloced '(loghandle)': 352 at ffff8b5b850b6a00. 00000100:00000010:1.0:1773941414.291304:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60a80. 02000000:00000010:1.0:1773941414.291312:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7a00. 00000100:00000010:1.0:1773941414.291319:0:12017:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b885de180. 02000000:00000010:1.0:1773941414.291329:0:12017:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b850b7000. 00000400:00000200:1.0:1773941414.291333:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941414.291337:0:12017:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034c08. 00000400:00000010:1.0:1773941414.291341:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f12a8. 00000100:00000200:1.0:1773941414.291344:0:12017:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241141632, portal 25 00000400:00000010:1.0:1773941414.291349:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1000. 00000100:00000200:1.0:1773941414.291352:0:12017:0:(niobuf.c:94:ptl_send_buf()) Sending 504 bytes to portal 26, xid 1860112241141632, offset 0 00000400:00000200:1.0:1773941414.291358:0:12017:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941414.291362:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.291366:0:12017:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941414.291369:0:12017:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941414.291372:0:12017:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941414.291376:0:12017:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941414.291379:0:12017:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941414.291383:0:12017:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [86:86:256:4294967295] 192.168.204.143@tcp LPNI seq info [86:86:8:4294967295] 00000400:00000200:1.0:1773941414.291387:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.291408:0:12017:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941414.291413:0:12017:0:(socklnd_cb.c:1001:ksocknal_send()) sending 504 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941414.291418:0:12017:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024bf00. 00000800:00000200:1.0:1773941414.291422:0:12017:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941414.291427:0:12017:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941414.291430:0:12017:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024bf00 type 1, nob 600 niov 1 nkiov 1 00000400:00000200:1.0:1773941414.291446:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:2.0:1773941414.291516:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941414.291525:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024bf00. 00000400:00000200:2.0:1773941414.291529:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.291534:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941414.291538:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1000 00000400:00000010:2.0:1773941414.291540:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1000. 00000100:00000200:2.0:1773941414.291547:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60a80 x1860112241141632/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941430 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941414.292468:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.292473:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941414.292479:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.292482:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.292485:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.292489:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 272 into portal 25 MB=0x69bc321602780 00000400:00000200:2.0:1773941414.292493:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.292497:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 272/272 into md 0x7d5 [1] + 224 00000400:00000200:2.0:1773941414.292507:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.292511:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941414.292517:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60a80 x1860112241141632/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941414.292556:0:12017:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034c08. 00000400:00000200:0.0:1773941414.292561:0:12017:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f12a8 00000400:00000010:0.0:1773941414.292562:0:12017:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f12a8. 00000100:00000200:0.0:1773941414.292568:0:12017:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60a80 x1860112241141632/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941414.292576:0:12017:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60a80 x1860112241141632/t0(0) o501->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1773941414.292588:0:12017:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b885de180. 02000000:00000010:0.0:1773941414.292593:0:12017:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b850b7000. 02000000:00000010:0.0:1773941414.292595:0:12017:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7a00. 00000100:00000010:0.0:1773941414.292597:0:12017:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60a80. 00000040:00000010:0.0:1773941414.292603:0:12017:0:(llog.c:412:llog_init_handle()) kmalloced '(llh)': 8192 at ffff8b5b850d8000. 00000100:00000010:0.0:1773941414.292607:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b850a8a80. 02000000:00000010:0.0:1773941414.292613:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b88908a00. 00000100:00000010:0.0:1773941414.292624:0:12017:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b890649c0. 02000000:00000010:0.0:1773941414.292643:0:12017:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at ffff8b5ba0c00000. 00000400:00000200:0.0:1773941414.292648:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941414.292652:0:12017:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941414.292657:0:12017:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at ffff8b5ba0b6e480. 00000100:00000200:0.0:1773941414.292661:0:12017:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112241141760, portal 25 00000400:00000010:0.0:1773941414.292666:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c198. 00000100:00000200:0.0:1773941414.292669:0:12017:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112241141760, offset 0 00000400:00000200:0.0:1773941414.292675:0:12017:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941414.292678:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941414.292681:0:12017:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941414.292683:0:12017:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941414.292685:0:12017:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941414.292687:0:12017:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941414.292688:0:12017:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941414.292691:0:12017:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [87:87:256:4294967295] 192.168.204.143@tcp LPNI seq info [87:87:8:4294967295] 00000400:00000200:0.0:1773941414.292694:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941414.292698:0:12017:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941414.292701:0:12017:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941414.292725:0:12017:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101500. 00000800:00000200:0.0:1773941414.292730:0:12017:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941414.292735:0:12017:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941414.292738:0:12017:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101500 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:0.0:1773941414.292750:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:2.0:1773941414.292792:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941414.292795:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101500. 00000400:00000200:2.0:1773941414.292800:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.292804:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941414.292808:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c198 00000400:00000010:2.0:1773941414.292810:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c198. 00000100:00000200:2.0:1773941414.292816:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b850a8a80 x1860112241141760/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941430 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941414.293600:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.293607:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941414.293613:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.293649:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.293654:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.293657:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 8416 into portal 25 MB=0x69bc321602800 00000400:00000200:3.0:1773941414.293662:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.293666:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 8416/8416 into md 0x7dd [4] + 224 00000400:00000200:3.0:1773941414.293678:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941414.293682:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941414.293689:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b850a8a80 x1860112241141760/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941414.293805:0:12017:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941414.293809:0:12017:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6e480 00000400:00000010:0.0:1773941414.293810:0:12017:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at ffff8b5ba0b6e480. 00000100:00000200:0.0:1773941414.293815:0:12017:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b850a8a80 x1860112241141760/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941414.293821:0:12017:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b850a8a80 x1860112241141760/t0(0) o503->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941430 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1773941414.293841:0:12017:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b890649c0. 02000000:00000010:0.0:1773941414.293849:0:12017:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at ffff8b5ba0c00000. 02000000:00000010:0.0:1773941414.293861:0:12017:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b88908a00. 00000100:00000010:0.0:1773941414.293865:0:12017:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b850a8a80. 00000040:00000010:0.0:1773941414.293873:0:12017:0:(llog.c:945:llog_process_or_fork()) kmalloced '(lpi)': 96 at ffff8b5b85ef6720. 00000020:00000010:3.0:1773941414.294041:0:12042:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b91df5800. 00000020:00000010:3.0:1773941414.294048:0:12042:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b91df4600. 00000040:00000010:3.0:1773941414.294052:0:12042:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b91df4a00. 00000020:00000010:3.0:1773941414.294058:0:12042:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b98ac0000. 00000020:00000010:3.0:1773941414.294064:0:12042:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b8849a840. 00000020:00000010:3.0:1773941414.294069:0:12042:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b91df5a00. 80000000:00000010:3.0:1773941414.294074:0:12042:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc93c0. 40000000:00000010:3.0:1773941414.294080:0:12042:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b850685c0. 00100000:00000010:3.0:1773941414.294085:0:12042:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b98ac0800. 00040000:00000010:3.0:1773941414.294091:0:12042:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b884be200. 00040000:00000010:3.0:1773941414.294096:0:12042:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b98ac7000. 00040000:00000010:3.0:1773941414.294102:0:12042:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b84e8e400. 00000004:00000010:3.0:1773941414.294107:0:12042:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b98ac5000. 00000004:00000010:3.0:1773941414.294114:0:12042:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5ba0b8d000. 00000004:00000010:3.0:1773941414.294120:0:12042:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b98ac6800. 00002000:00000010:3.0:1773941414.294126:0:12042:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b90262000. 00000004:00000010:3.0:1773941414.294132:0:12042:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5b90261800. 00000020:00000010:3.0:1773941414.294146:0:12042:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b850684c0. 00000040:00000010:3.0:1773941414.294161:0:12042:0:(llog.c:570:llog_process_thread()) kmalloced '(buf)': 8192 at ffff8b5b90d44000. 00000040:00000010:3.0:1773941414.294670:0:12042:0:(llog.c:891:llog_process_thread()) kvfree 'buf': 8192 at ffff8b5b90d44000. 00000020:00000010:3.0:1773941414.294678:0:12042:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b850684c0. 00000020:00000010:3.0:1773941414.294689:0:12042:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b91df4600. 00000040:00000010:3.0:1773941414.294693:0:12042:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b91df4a00. 00000020:00000010:3.0:1773941414.294697:0:12042:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b98ac0000. 00000020:00000010:3.0:1773941414.294701:0:12042:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b8849a840. 00000020:00000010:3.0:1773941414.294750:0:12042:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b91df5a00. 80000000:00000010:3.0:1773941414.294756:0:12042:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc93c0. 40000000:00000010:3.0:1773941414.294760:0:12042:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b850685c0. 00100000:00000010:3.0:1773941414.294765:0:12042:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b98ac0800. 00040000:00000010:3.0:1773941414.294769:0:12042:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b884be200. 00040000:00000010:3.0:1773941414.294774:0:12042:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b98ac7000. 00040000:00000010:3.0:1773941414.294778:0:12042:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b84e8e400. 00000004:00000010:3.0:1773941414.294783:0:12042:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b98ac5000. 00000004:00000010:3.0:1773941414.294788:0:12042:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5ba0b8d000. 00000004:00000010:3.0:1773941414.294793:0:12042:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b98ac6800. 00002000:00000010:3.0:1773941414.294797:0:12042:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5b90262000. 00000004:00000010:3.0:1773941414.294801:0:12042:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5b90261800. 00000020:00000010:3.0:1773941414.294805:0:12042:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b91df5800. 00000040:00000010:1.0:1773941414.294838:0:12017:0:(llog.c:986:llog_process_or_fork()) kfreed 'lpi': 96 at ffff8b5b85ef6720. 00000040:00000010:1.0:1773941414.294845:0:12017:0:(llog.c:70:llog_free_handle()) kvfree 'loghandle->lgh_hdr': 8192 at ffff8b5b850d8000. 00000040:00000010:1.0:1773941414.294856:0:12017:0:(llog.c:72:llog_free_handle()) kfreed 'loghandle': 352 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941414.294861:0:12017:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b7800. 00000040:00000010:1.0:1773941414.294865:0:12017:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6c00. 00000020:00000010:1.0:1773941414.294868:0:12017:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b88954800. 00000020:00000010:1.0:1773941414.294872:0:12017:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7c00. 80000000:00000010:1.0:1773941414.294875:0:12017:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f99c0. 00040000:00000010:1.0:1773941414.294879:0:12017:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6400. 20000000:00000010:1.0:1773941414.294883:0:12017:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b7400. 00000020:00000010:1.0:1773941414.294886:0:12017:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b6e00. 10000000:00000010:1.0:1773941414.294889:0:12017:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0f00. 00000020:00000010:1.0:1773941414.294904:0:12017:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b84d8b400. 00000040:00000010:1.0:1773941414.294908:0:12017:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b84d8b200. 00000020:00000010:1.0:1773941414.294911:0:12017:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b98ac5800. 00000020:00000010:1.0:1773941414.294915:0:12017:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b902a0800. 00040000:00000010:1.0:1773941414.294919:0:12017:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b884be400. 00040000:00000010:1.0:1773941414.294922:0:12017:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b902a1a00. 00000020:00000010:1.0:1773941414.294925:0:12017:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84d8a000. 00000020:00000010:1.0:1773941414.294929:0:12017:0:(obd_mount.c:82:lustre_process_log()) kfreed 'lcfg': 184 at ffff8b5b85028d80. 00000020:00000010:1.0:1773941414.294932:0:12017:0:(obd_mount.c:84:lustre_process_log()) kfreed 'bufs': 104 at ffff8b5b9020a700. 00000080:00000010:1.0:1773941414.294939:0:12017:0:(llite_lib.c:1502:ll_fill_super()) kmalloced '(dt)': 33 at ffff8b5b850b0f00. 00000080:00000010:1.0:1773941414.294943:0:12017:0:(llite_lib.c:1508:ll_fill_super()) kmalloced '(md)': 33 at ffff8b5b850b07c0. 00000080:00000010:1.0:1773941414.294949:0:12017:0:(llite_lib.c:348:client_common_fill_super()) kmalloced '(data)': 192 at ffff8b5b885dee40. 00000080:00000010:1.0:1773941414.294952:0:12017:0:(llite_lib.c:352:client_common_fill_super()) kmalloced '(osfs)': 144 at ffff8b5b885ded80. 00000020:00000010:1.0:1773941414.294958:0:12017:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5b88954800. 00000020:00000010:1.0:1773941414.294985:0:12017:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5b88956800. 00000400:00000200:1.0:1773941414.294996:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.295001:0:12017:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000010:1.0:1773941414.295015:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60000. 02000000:00000010:1.0:1773941414.295023:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0735c00. 40000000:00000010:1.0:1773941414.295055:0:12017:0:(fid_request.c:432:client_fid_init()) kmalloced '(cli->cl_seq)': 224 at ffff8b5b9024be00. 40000000:00000010:1.0:1773941414.295059:0:12017:0:(fid_request.c:436:client_fid_init()) kmalloced '(prefix)': 133 at ffff8b5b885decc0. 40000000:00000010:1.0:1773941414.295124:0:12017:0:(fid_request.c:444:client_fid_init()) kfreed 'prefix': 133 at ffff8b5b885decc0. 80000000:00000010:1.0:1773941414.295129:0:12017:0:(fld_request.c:150:fld_client_add_target()) kmalloced '(target)': 40 at ffff8b5b850b0340. 00020000:00000010:1.0:1773941414.295141:0:12017:0:(lu_tgt_descs.c:102:lu_qos_add_tgt()) kmalloced '(svr)': 104 at ffff8b5b90edbe00. 00000100:00000010:1.0:1773941414.295148:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63b80. 02000000:00000010:1.0:1773941414.295154:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b9024b200. 00800000:00000010:1.0:1773941414.295180:0:12017:0:(lmv_obd.c:1417:lmv_statfs()) kmalloced '(temp)': 144 at ffff8b5b885decc0. 00000100:00000010:1.0:1773941414.295185:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60700. 02000000:00000010:1.0:1773941414.295190:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b9024b500. 00000100:00000010:1.0:1773941414.295194:0:12017:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b885def00. 00000008:00000010:1.0:1773941414.295212:0:2351:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5b90d8ce18. 00020000:00000010:1.0:1773941414.295216:0:2351:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a10df00. 00000008:00000010:1.0:1773941414.295222:0:2351:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5ba0b99340. 00020000:00000010:1.0:1773941414.295226:0:2351:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5ba0b9f840. 02000000:00000010:1.0:1773941414.295234:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735400. 00000400:00000200:1.0:1773941414.295238:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941414.295241:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034a50. 00000400:00000010:1.0:1773941414.295246:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1088. 00000100:00000200:1.0:1773941414.295250:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241141888, portal 10 00000020:00000010:1.0:1773941414.295266:0:2351:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff8b5b85e54000. 00000400:00000010:1.0:1773941414.295273:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1330. 00000100:00000200:1.0:1773941414.295276:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1860112241141888, offset 0 00000400:00000200:1.0:1773941414.295282:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941414.295287:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.295293:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941414.295296:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941414.295299:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941414.295302:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941414.295305:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941414.295308:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [88:88:256:4294967295] 192.168.204.143@tcp LPNI seq info [88:88:8:4294967295] 00000400:00000200:1.0:1773941414.295313:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.295322:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941414.295327:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941414.295331:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b400. 00000800:00000200:1.0:1773941414.295340:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941414.295345:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941414.295348:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b400 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:1.0:1773941414.295364:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000008:00000010:1.0:1773941414.295403:0:2353:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5b90d8c2f0. 00020000:00000010:1.0:1773941414.295408:0:2353:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a10dcc0. 00000008:00000010:1.0:1773941414.295413:0:2353:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5ba0b98b00. 00020000:00000010:1.0:1773941414.295417:0:2353:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5ba0b9e1c0. 00000008:00000010:1.0:1773941414.295430:0:2352:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5b90d8a178. 00020000:00000010:1.0:1773941414.295434:0:2352:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a10d840. 00000008:00000010:1.0:1773941414.295438:0:2352:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5ba0b98580. 00000800:00000200:3.0:1773941414.295439:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00020000:00000010:1.0:1773941414.295441:0:2352:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5ba0b9fa80. 00000800:00000010:3.0:1773941414.295444:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b400. 00000400:00000200:3.0:1773941414.295450:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941414.295456:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941414.295460:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1330 00000400:00000010:3.0:1773941414.295462:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1330. 00000100:00000200:3.0:1773941414.295471:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60000 x1860112241141888/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941424 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941414.296158:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.296164:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941414.296170:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.296172:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.296176:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.296186:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 10 MB=0x69bc321602880 00000400:00000200:3.0:1773941414.296190:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941414.296195:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.143@tcp of length 416/416 into md 0x7e5 [1] + 0 00000400:00000200:3.0:1773941414.296206:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941414.296210:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941414.296217:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60000 x1860112241141888/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941424 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941414.296252:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034a50. 00000400:00000200:1.0:1773941414.296257:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1088 00000400:00000010:1.0:1773941414.296259:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1088. 00000100:00000200:1.0:1773941414.296266:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60000 x1860112241141888/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941424 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941414.296278:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60000 x1860112241141888/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941424 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1773941414.296297:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735400. 02000000:00000010:1.0:1773941414.296301:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0735c00. 00000100:00000010:1.0:1773941414.296304:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60000. 00000020:00000010:2.0:1773941414.297622:0:12041:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at ffff8b5b845adfc0. 00000100:00000200:2.0:1773941414.297634:0:12041:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.143@tcp->12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941414.297637:0:12041:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.297640:0:12041:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.143@tcp primary NID 192.168.204.143@tcp rc 0 00000100:00000010:2.0:1773941414.297655:0:12041:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021fb80. 02000000:00000010:2.0:1773941414.297661:0:12041:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b83f09000. 00000020:00000010:2.0:1773941414.297674:0:12041:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at ffff8b5b845adfc0. 02000000:00000010:1.0:1773941414.297700:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735c00. 00000400:00000200:1.0:1773941414.297719:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941414.297722:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034a50. 00000400:00000010:1.0:1773941414.297726:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1088. 00000100:00000200:1.0:1773941414.297729:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241142272, portal 10 00000400:00000010:1.0:1773941414.297732:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f13b8. 00000100:00000200:1.0:1773941414.297734:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1860112241142272, offset 0 00000400:00000200:1.0:1773941414.297738:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941414.297741:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.297750:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941414.297753:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941414.297754:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941414.297756:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941414.297757:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941414.297759:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [89:89:256:4294967295] 192.168.204.143@tcp LPNI seq info [89:89:8:4294967295] 00000400:00000200:1.0:1773941414.297762:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941414.297768:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941414.297771:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941414.297774:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024bb00. 00000800:00000200:1.0:1773941414.297778:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941414.297782:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941414.297785:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024bb00 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:1.0:1773941414.297794:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941414.297839:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941414.297843:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024bb00. 00000400:00000200:3.0:1773941414.297847:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941414.297852:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941414.297856:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f13b8 00000400:00000010:3.0:1773941414.297858:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f13b8. 00000100:00000200:3.0:1773941414.297873:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021fb80 x1860112241142272/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941424 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'lctl.0' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1773941414.298421:0:12043:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at ffff8b5ba039cc40. 00000020:00000010:0.0:1773941414.298426:0:12043:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at ffff8b5ba039cc40. 00000400:00000200:2.0:1773941414.298568:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.298575:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941414.298582:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.298586:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.298590:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.298594:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 10 MB=0x69bc321602a00 00000400:00000200:2.0:1773941414.298599:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941414.298605:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.143@tcp of length 416/416 into md 0x7ed [1] + 0 00000400:00000200:2.0:1773941414.298616:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941414.298621:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941414.298629:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021fb80 x1860112241142272/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941424 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'lctl.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941414.298692:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034a50. 00000400:00000200:1.0:1773941414.298697:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1088 00000400:00000010:1.0:1773941414.298699:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1088. 00000100:00000200:1.0:1773941414.298726:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021fb80 x1860112241142272/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941424 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'lctl.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941414.298737:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021fb80 x1860112241142272/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941424 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'lctl.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1773941414.298758:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735c00. 02000000:00000010:1.0:1773941414.298764:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b83f09000. 00000100:00000010:1.0:1773941414.298768:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021fb80. 00000100:00000010:2.0:1773941415.135818:0:761:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8b5b84cfb480. 00000100:00000010:3.0:1773941415.135819:0:417:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8b5b84cfba80. 00000100:00000010:3.0:1773941419.424006:0:37:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90276300. 02000000:00000010:3.0:1773941419.424026:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b884be200. 00000100:00000010:3.0:1773941419.424068:0:37:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275500. 02000000:00000010:3.0:1773941419.424074:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90261800. 00000008:00000010:2.0:1773941419.424081:0:2355:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f02ca0. 00020000:00000010:2.0:1773941419.424088:0:2355:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a2fd180. 00000008:00000010:2.0:1773941419.424093:0:2355:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5b885b29a0. 00020000:00000010:2.0:1773941419.424096:0:2355:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5b84cbfa80. 02000000:00000010:2.0:1773941419.424110:0:2355:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b84fee600. 00000008:00000010:3.0:1773941419.424117:0:2354:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f0e468. 00000400:00000200:2.0:1773941419.424117:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941419.424122:0:2355:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af99a0. 00020000:00000010:3.0:1773941419.424124:0:2354:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b893820c0. 00000400:00000010:2.0:1773941419.424129:0:2355:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f347f8. 00000008:00000010:3.0:1773941419.424131:0:2354:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5ba0c338c0. 00020000:00000010:3.0:1773941419.424136:0:2354:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5ba0c34d80. 00000100:00000200:2.0:1773941419.424136:0:2355:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241142400, portal 25 00000400:00000010:2.0:1773941419.424146:0:2355:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34f68. 02000000:00000010:1.0:1773941419.424146:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735c00. 00000100:00000200:2.0:1773941419.424153:0:2355:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241142400, offset 0 00000400:00000200:1.0:1773941419.424162:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941419.424163:0:2355:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941419.424175:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034a50. 00000400:00000200:2.0:1773941419.424182:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941419.424188:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1088. 00000400:00000200:2.0:1773941419.424192:0:2355:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941419.424197:0:2355:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000200:1.0:1773941419.424198:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241142528, portal 10 00000400:00000010:1.0:1773941419.424207:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1aa0. 00000400:00000200:2.0:1773941419.424218:0:2355:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941419.424227:0:2355:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000100:00000200:1.0:1773941419.424229:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1860112241142528, offset 0 00000400:00000200:2.0:1773941419.424230:0:2355:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941419.424237:0:2355:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [90:90:256:4294967295] 192.168.204.143@tcp LPNI seq info [90:90:8:4294967295] 00000400:00000200:2.0:1773941419.424242:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941419.424249:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941419.424256:0:2355:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000400:00000200:1.0:1773941419.424264:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:2.0:1773941419.424270:0:2355:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941419.424275:0:2355:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0800. 00000400:00000200:1.0:1773941419.424279:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000800:00000200:2.0:1773941419.424285:0:2355:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000400:00000200:1.0:1773941419.424286:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941419.424318:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000800:00000200:2.0:1773941419.424319:0:2355:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000400:00000200:1.0:1773941419.424327:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000800:00000200:2.0:1773941419.424328:0:2355:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1773941419.424334:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941419.424341:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [91:91:255:4294967295] 192.168.204.143@tcp LPNI seq info [91:91:7:4294967295] 00000400:00000200:1.0:1773941419.424346:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941419.424353:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941419.424362:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000100:00000010:2.0:1773941419.424364:0:2355:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b5ba0166d20. 00000800:00000200:1.0:1773941419.424373:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941419.424384:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b600. 00000800:00000200:1.0:1773941419.424391:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000100:00000010:2.0:1773941419.424394:0:2355:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b5ba0166000. 00000800:00000200:1.0:1773941419.424400:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941419.424405:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b600 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:1.0:1773941419.424414:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941419.424446:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941419.424458:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0800. 00000400:00000200:3.0:1773941419.424464:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941419.424471:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941419.424475:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34f68 00000400:00000010:3.0:1773941419.424478:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34f68. 00000100:00000200:3.0:1773941419.424487:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90276300 x1860112241142400/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941435 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1773941419.424602:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941419.424604:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b600. 00000400:00000200:3.0:1773941419.424607:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941419.424610:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941419.424613:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1aa0 00000400:00000010:3.0:1773941419.424614:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1aa0. 00000100:00000200:3.0:1773941419.424618:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275500 x1860112241142528/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941429 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941419.425737:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941419.425741:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941419.425746:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941419.425749:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941419.425752:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941419.425755:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321602a80 00000400:00000200:2.0:1773941419.425759:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941419.425762:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x7f5 [1] + 224 00000400:00000200:2.0:1773941419.425772:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941419.425775:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:2.0:1773941419.425782:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90276300 x1860112241142400/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941435 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941419.425825:0:2355:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af99a0. 00000400:00000200:3.0:1773941419.425830:0:2355:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f347f8 00000400:00000010:3.0:1773941419.425832:0:2355:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f347f8. 00000100:00000200:3.0:1773941419.425839:0:2355:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90276300 x1860112241142400/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941435 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941419.425849:0:2355:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90276300 x1860112241142400/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941435 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1773941419.425871:0:2355:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b84fee600. 02000000:00000010:3.0:1773941419.425875:0:2355:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b884be200. 00000100:00000010:3.0:1773941419.425878:0:2355:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90276300. 00000400:00000200:3.0:1773941419.428976:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.428985:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941419.428993:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.428997:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.429001:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.429005:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 10 MB=0x69bc321602b00 00000400:00000200:3.0:1773941419.429011:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.429016:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.143@tcp of length 416/416 into md 0x805 [1] + 0 00000400:00000200:3.0:1773941419.429031:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941419.429036:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941419.429049:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275500 x1860112241142528/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941429 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941419.429161:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034a50. 00000400:00000200:1.0:1773941419.429167:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1088 00000400:00000010:1.0:1773941419.429169:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1088. 00000100:00000200:1.0:1773941419.429176:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275500 x1860112241142528/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941429 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941419.429186:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275500 x1860112241142528/t0(0) o38->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941429 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1773941419.429493:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735c00. 02000000:00000010:2.0:1773941419.429501:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90261800. 00000100:00000010:2.0:1773941419.429505:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275500. 02000000:00000010:3.0:1773941419.429572:0:12017:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90263400. 00000400:00000200:3.0:1773941419.429579:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941419.429582:0:12017:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87c60. 00000400:00000010:3.0:1773941419.429587:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a660. 02000000:00000010:0.0:1773941419.429592:0:2353:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0bf1000. 00000100:00000200:3.0:1773941419.429593:0:12017:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241142144, portal 10 00000400:00000200:0.0:1773941419.429601:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000010:3.0:1773941419.429604:0:12017:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff8b5b90d44000. 00000400:00000010:0.0:1773941419.429606:0:2353:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941419.429612:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000400:00000010:3.0:1773941419.429614:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a198. 00000100:00000200:0.0:1773941419.429616:0:2353:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241142016, portal 10 00000100:00000200:3.0:1773941419.429619:0:12017:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1860112241142144, offset 0 00000020:00000010:0.0:1773941419.429624:0:2353:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff8b5b850dc000. 00000400:00000200:3.0:1773941419.429630:0:12017:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941419.429632:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c660. 00000400:00000200:3.0:1773941419.429638:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000200:0.0:1773941419.429638:0:2353:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1860112241142016, offset 0 00000400:00000200:3.0:1773941419.429648:0:12017:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941419.429650:0:2353:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941419.429656:0:12017:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941419.429660:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.429664:0:12017:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941419.429668:0:2353:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941419.429671:0:12017:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941419.429675:0:2353:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941419.429678:0:12017:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941419.429682:0:2353:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941419.429686:0:12017:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [92:92:256:4294967295] 192.168.204.143@tcp LPNI seq info [92:92:8:4294967295] 00000400:00000200:0.0:1773941419.429690:0:2353:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941419.429696:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941419.429697:0:2353:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:3.0:1773941419.429736:0:12017:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000400:00000200:0.0:1773941419.429740:0:2353:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [93:93:256:4294967295] 192.168.204.143@tcp LPNI seq info [93:93:8:4294967295] 00000800:00000200:3.0:1773941419.429746:0:12017:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941419.429753:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000010:3.0:1773941419.429754:0:12017:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b884be200. 00000800:00000200:3.0:1773941419.429767:0:12017:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000400:00000200:0.0:1773941419.429769:0:2353:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:3.0:1773941419.429781:0:12017:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941419.429781:0:2353:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941419.429786:0:2353:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00000800:00000200:3.0:1773941419.429788:0:12017:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b884be200 type 1, nob 320 niov 1 nkiov 1 00000800:00000200:0.0:1773941419.429794:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941419.429799:0:2353:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941419.429808:0:2353:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1773941419.429812:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941419.429817:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:2.0:1773941419.429900:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941419.429903:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b884be200. 00000400:00000200:2.0:1773941419.429906:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941419.429908:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941419.429911:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a198 00000400:00000010:2.0:1773941419.429912:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a198. 00000100:00000200:2.0:1773941419.429918:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60700 x1860112241142144/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941435 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1773941419.429965:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941419.429968:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000400:00000200:2.0:1773941419.429971:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941419.429975:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941419.429977:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c660 00000400:00000010:2.0:1773941419.429979:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c660. 00000100:00000200:2.0:1773941419.429984:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63b80 x1860112241142016/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941435 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941419.431485:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.431490:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941419.431494:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.431496:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.431499:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.431507:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 368 into portal 10 MB=0x69bc321602900 00000400:00000200:3.0:1773941419.431510:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.431512:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.143@tcp of length 368/368 into md 0x815 [1] + 224 00000400:00000200:3.0:1773941419.431520:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941419.431522:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941419.431528:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63b80 x1860112241142016/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941435 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941419.431584:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.431588:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941419.431591:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941419.431592:0:2353:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:3.0:1773941419.431593:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.431598:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941419.431599:0:2353:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000200:3.0:1773941419.431601:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 368 into portal 10 MB=0x69bc321602980 00000400:00000010:0.0:1773941419.431603:0:2353:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000400:00000200:3.0:1773941419.431607:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.431611:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.143@tcp of length 368/368 into md 0x80d [1] + 224 00000100:00000200:0.0:1773941419.431615:0:2353:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63b80 x1860112241142016/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941435 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941419.431627:0:2353:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63b80 x1860112241142016/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941435 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941419.431636:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941419.431640:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941419.431645:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60700 x1860112241142144/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941435 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1773941419.431654:0:2353:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0bf1000. 02000000:00000010:0.0:1773941419.431661:0:2353:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b9024b200. 00000100:00000010:0.0:1773941419.431666:0:2353:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63b80. 00000400:00000010:2.0:1773941419.431699:0:12017:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87c60. 00000400:00000200:2.0:1773941419.431724:0:12017:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a660 00000400:00000010:2.0:1773941419.431727:0:12017:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a660. 00000100:00000200:2.0:1773941419.431742:0:12017:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60700 x1860112241142144/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941435 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941419.431757:0:12017:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60700 x1860112241142144/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941435 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1773941419.431771:0:12017:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff8b5ba0c74000. 00000100:00000010:2.0:1773941419.431780:0:12017:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b885def00. 02000000:00000010:2.0:1773941419.431786:0:12017:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90263400. 02000000:00000010:2.0:1773941419.431789:0:12017:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b9024b500. 00000100:00000010:2.0:1773941419.431792:0:12017:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60700. 00800000:00000010:2.0:1773941419.431797:0:12017:0:(lmv_obd.c:1477:lmv_statfs()) kfreed 'temp': 144 at ffff8b5b885decc0. 00000100:00000010:2.0:1773941419.431806:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021ca80. 02000000:00000010:2.0:1773941419.431815:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b84fee600. 00000100:00000010:2.0:1773941419.431830:0:12017:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0cfb900. 02000000:00000010:2.0:1773941419.431843:0:12017:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b83f09800. 00000400:00000200:2.0:1773941419.431848:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941419.431851:0:12017:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af97e8. 00000400:00000010:2.0:1773941419.431855:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34d48. 00000100:00000200:2.0:1773941419.431859:0:12017:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241142656, portal 10 00000400:00000010:2.0:1773941419.431863:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34bb0. 00000100:00000200:2.0:1773941419.431865:0:12017:0:(niobuf.c:94:ptl_send_buf()) Sending 440 bytes to portal 12, xid 1860112241142656, offset 0 00000400:00000200:2.0:1773941419.431871:0:12017:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941419.431875:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941419.431879:0:12017:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941419.431882:0:12017:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941419.431884:0:12017:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941419.431887:0:12017:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941419.431894:0:12017:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941419.431897:0:12017:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [94:94:256:4294967295] 192.168.204.143@tcp LPNI seq info [94:94:8:4294967295] 00000400:00000200:2.0:1773941419.431901:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941419.431908:0:12017:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941419.431912:0:12017:0:(socklnd_cb.c:1001:ksocknal_send()) sending 440 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941419.431916:0:12017:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0e00. 00000800:00000200:2.0:1773941419.431921:0:12017:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941419.431928:0:12017:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941419.431931:0:12017:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0e00 type 1, nob 536 niov 1 nkiov 1 00000400:00000200:2.0:1773941419.431946:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941419.432005:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941419.432009:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0e00. 00000400:00000200:3.0:1773941419.432013:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941419.432017:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941419.432020:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34bb0 00000400:00000010:3.0:1773941419.432022:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34bb0. 00000100:00000200:3.0:1773941419.432028:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021ca80 x1860112241142656/t0(0) o40->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941435 ref 3 fl Rpc:r/209/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941419.432767:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.432771:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941419.432779:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.432781:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.432783:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.432792:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 448 into portal 10 MB=0x69bc321602b80 00000400:00000200:3.0:1773941419.432796:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.432799:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.143@tcp of length 448/448 into md 0x825 [1] + 224 00000400:00000200:3.0:1773941419.432807:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941419.432810:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941419.432816:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021ca80 x1860112241142656/t0(0) o40->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941435 ref 2 fl Rpc:Qr/209/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941419.432873:0:12017:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af97e8. 00000400:00000200:2.0:1773941419.432878:0:12017:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34d48 00000400:00000010:2.0:1773941419.432880:0:12017:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34d48. 00000100:00000200:2.0:1773941419.432886:0:12017:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021ca80 x1860112241142656/t0(0) o40->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941435 ref 2 fl Rpc:RQ/209/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941419.432895:0:12017:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021ca80 x1860112241142656/t0(0) o40->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941435 ref 2 fl Rpc:RQU/209/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1773941419.432922:0:12017:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0cfb900. 00000002:00000200:2.0:1773941419.432927:0:12017:0:(mdc_request.c:134:mdc_get_root()) root fid=[0x200000007:0x1:0x0], last_committed=0 02000000:00000010:2.0:1773941419.432931:0:12017:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b83f09800. 02000000:00000010:2.0:1773941419.432935:0:12017:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b84fee600. 00000100:00000010:2.0:1773941419.432938:0:12017:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021ca80. 00000020:00000010:2.0:1773941419.432955:0:12017:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5b90f27000. 00000020:00000010:2.0:1773941419.432971:0:12017:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5b90f20800. 00000400:00000200:2.0:1773941419.432983:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941419.432986:0:12017:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000010:2.0:1773941419.432998:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021ca80. 02000000:00000010:2.0:1773941419.433004:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b83f09800. 00000080:00000010:2.0:1773941419.433042:0:12017:0:(llite_lib.c:738:client_common_fill_super()) kmalloced '(op_data)': 384 at ffff8b5b84fee600. 00000020:00000010:2.0:1773941419.433046:0:12017:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffff8b5b83f0a000. 02000000:00000010:1.0:1773941419.433050:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735400. 00000020:00000010:2.0:1773941419.433051:0:12017:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffff8b5b83f08800. 00000100:00000010:2.0:1773941419.433058:0:12017:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021d500. 00000400:00000200:1.0:1773941419.433060:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 02000000:00000010:2.0:1773941419.433063:0:12017:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b84fef800. 00000400:00000010:1.0:1773941419.433066:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034a50. 00000100:00000010:2.0:1773941419.433071:0:12017:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0cfb900. 00000400:00000010:1.0:1773941419.433074:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1088. 00000100:00000200:1.0:1773941419.433079:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241142784, portal 4 00000020:00000010:1.0:1773941419.433087:0:2351:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff8b5b85e56000. 00000400:00000010:1.0:1773941419.433099:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1e58. 00000100:00000200:1.0:1773941419.433104:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1860112241142784, offset 0 00000400:00000200:1.0:1773941419.433110:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 02000000:00000010:2.0:1773941419.433115:0:12017:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 131072 at ffff8b5b889a0000. 00000400:00000200:1.0:1773941419.433115:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941419.433122:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941419.433125:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000010:2.0:1773941419.433126:0:12017:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af97e8. 00000400:00000010:2.0:1773941419.433132:0:12017:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 632 at ffff8b5b83f0bc00. 00000400:00000200:1.0:1773941419.433132:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941419.433138:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000100:00000200:2.0:1773941419.433139:0:12017:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 131072 bytes, xid 1860112241142912, portal 10 00000400:00000200:1.0:1773941419.433144:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000010:2.0:1773941419.433146:0:12017:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34d48. 00000400:00000200:1.0:1773941419.433149:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000100:00000200:2.0:1773941419.433151:0:12017:0:(niobuf.c:94:ptl_send_buf()) Sending 448 bytes to portal 23, xid 1860112241142912, offset 0 00000400:00000200:1.0:1773941419.433156:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [95:95:256:4294967295] 192.168.204.143@tcp LPNI seq info [95:95:8:4294967295] 00000400:00000200:2.0:1773941419.433162:0:12017:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941419.433166:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941419.433170:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941419.433175:0:12017:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941419.433178:0:12017:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941419.433181:0:12017:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941419.433183:0:12017:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941419.433193:0:12017:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941419.433196:0:12017:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [96:96:256:4294967295] 192.168.204.143@tcp LPNI seq info [96:96:8:4294967295] 00000400:00000200:2.0:1773941419.433199:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941419.433205:0:12017:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941419.433211:0:12017:0:(socklnd_cb.c:1001:ksocknal_send()) sending 448 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941419.433215:0:12017:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0d00. 00000400:00000200:1.0:1773941419.433220:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941419.433225:0:12017:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941419.433231:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000200:2.0:1773941419.433236:0:12017:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000010:1.0:1773941419.433239:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b700. 00000800:00000200:2.0:1773941419.433240:0:12017:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0d00 type 1, nob 544 niov 1 nkiov 1 00000800:00000200:1.0:1773941419.433250:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941419.433256:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000400:00000200:2.0:1773941419.433260:0:12017:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:1.0:1773941419.433261:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b700 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:1.0:1773941419.433270:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000800:00000200:3.0:1773941419.433285:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941419.433288:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0d00. 00000400:00000200:3.0:1773941419.433291:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941419.433303:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941419.433305:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34d48 00000400:00000010:3.0:1773941419.433307:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34d48. 00000100:00000200:3.0:1773941419.433311:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021d500 x1860112241142912/t0(0) o33->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941435 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:0 00000800:00000200:3.0:1773941419.433341:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941419.433343:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b700. 00000400:00000200:3.0:1773941419.433345:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941419.433347:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941419.433348:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1e58 00000400:00000010:3.0:1773941419.433350:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1e58. 00000100:00000200:3.0:1773941419.433353:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021ca80 x1860112241142784/t0(0) o8->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941429 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941419.435017:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.435028:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941419.435035:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.435038:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.435050:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.435053:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 4 MB=0x69bc321602c00 00000400:00000200:3.0:1773941419.435057:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.435061:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.143@tcp of length 416/416 into md 0x835 [1] + 0 00000400:00000200:3.0:1773941419.435073:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941419.435076:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941419.435085:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021ca80 x1860112241142784/t0(0) o8->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941429 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941419.435126:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034a50. 00000400:00000200:1.0:1773941419.435131:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1088 00000400:00000010:1.0:1773941419.435133:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1088. 00000100:00000200:1.0:1773941419.435138:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021ca80 x1860112241142784/t0(0) o8->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941429 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941419.435145:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021ca80 x1860112241142784/t0(0) o8->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941429 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1773941419.435192:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735400. 02000000:00000010:1.0:1773941419.435197:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b83f09800. 00000100:00000010:1.0:1773941419.435201:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021ca80. 00000400:00000200:3.0:1773941419.440609:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.440618:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941419.440626:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.440631:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.440637:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.440640:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 464 into portal 10 MB=0x69bc321602c80 00000400:00000200:3.0:1773941419.440645:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941419.440650:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.143@tcp of length 464/464 into md 0x83d [32] + 224 00000400:00000200:3.0:1773941419.440742:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941419.440749:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000200:3.0:1773941419.440761:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021d500 x1860112241142912/t0(0) o33->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941435 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:0 00000400:00000010:2.0:1773941419.440860:0:12017:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af97e8. 00000400:00000200:2.0:1773941419.440865:0:12017:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f0bc00 00000400:00000010:2.0:1773941419.440867:0:12017:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 632 at ffff8b5b83f0bc00. 00000100:00000200:2.0:1773941419.440873:0:12017:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021d500 x1860112241142912/t0(0) o33->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941435 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:0 00000100:00000200:2.0:1773941419.440879:0:12017:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021d500 x1860112241142912/t0(0) o33->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941435 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:0 00000100:00000010:2.0:1773941419.440903:0:12017:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0cfb900. 00000002:00000200:2.0:1773941419.440910:0:12017:0:(mdc_request.c:172:mdc_getattr_common()) mode: 40755 00000080:00000010:2.0:1773941419.440914:0:12017:0:(llite_lib.c:753:client_common_fill_super()) kfreed 'op_data': 384 at ffff8b5b84fee600. 00000020:00000010:2.0:1773941419.440923:0:12017:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8b5b8509c2d0. 00000020:00000010:2.0:1773941419.440927:0:12017:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84fee600. 00000020:00000010:2.0:1773941419.440930:0:12017:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b84feea00. 00000020:00000010:2.0:1773941419.440932:0:12017:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0166a20. 00000008:00000010:2.0:1773941419.440937:0:12017:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f04e18. 00020000:00000010:2.0:1773941419.440940:0:12017:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a2fd780. 00008000:00000010:2.0:1773941419.440943:0:12017:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d77880. 00000020:00000010:2.0:1773941419.440947:0:12017:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84fef200. 00000008:00000010:2.0:1773941419.440949:0:12017:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5b885b3e40. 00020000:00000010:2.0:1773941419.440952:0:12017:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5b84cbc240. 00008000:00000010:2.0:1773941419.440965:0:12017:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5b82c37248. 00000080:00000010:2.0:1773941419.440969:0:12017:0:(vvp_dev.c:174:vvp_device_alloc()) kmalloced '(vdv)': 72 at ffff8b5ba0166ea0. 00000080:00000010:2.0:1773941419.440973:0:12017:0:(vvp_dev.c:182:vvp_device_alloc()) kmalloced '(site)': 576 at ffff8b5b83f0bc00. 00000020:00000010:2.0:1773941419.441001:0:12017:0:(lu_object.c:1154:lu_site_init()) kmalloced '(s->ls_bkts)': 18432 at ffff8b5ba0c88000. 00000020:00000010:2.0:1773941419.441008:0:12017:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5b83f09400. 00000020:00000010:2.0:1773941419.441011:0:12017:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 192 at ffff8b5ba0cfb900. 00000008:00000010:2.0:1773941419.441016:0:12017:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f06f90. 00020000:00000010:2.0:1773941419.441018:0:12017:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a2fd3c0. 00000080:00000010:2.0:1773941419.441021:0:12017:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b85eec348. 00000080:00000010:2.0:1773941419.441023:0:12017:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b98a88248. 00000008:00000010:2.0:1773941419.441025:0:12017:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5b885b3b80. 00020000:00000010:2.0:1773941419.441027:0:12017:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5b84cbc480. 00000080:00000010:2.0:1773941419.441029:0:12017:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b845f5690. 00000008:00000010:2.0:1773941419.441031:0:12017:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f037c8. 00020000:00000010:2.0:1773941419.441032:0:12017:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a2fd600. 00000080:00000010:2.0:1773941419.441034:0:12017:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b85eecb28. 00000080:00000010:2.0:1773941419.441035:0:12017:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b98a88b68. 00000008:00000010:2.0:1773941419.441037:0:12017:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5b885b3760. 00020000:00000010:2.0:1773941419.441038:0:12017:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5b84cbe640. 00000080:00000010:2.0:1773941419.441040:0:12017:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b845f55a0. 00000008:00000010:2.0:1773941419.441042:0:12017:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f06468. 00020000:00000010:2.0:1773941419.441043:0:12017:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a2fdd80. 00000080:00000010:2.0:1773941419.441044:0:12017:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b85eec150. 00000080:00000010:2.0:1773941419.441046:0:12017:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b98a8afe8. 00000008:00000010:2.0:1773941419.441047:0:12017:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5b885b2840. 00020000:00000010:2.0:1773941419.441049:0:12017:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5b84cbd8c0. 00000080:00000010:2.0:1773941419.441050:0:12017:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b845f5c30. 00000008:00000010:2.0:1773941419.441052:0:12017:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f05940. 00020000:00000010:2.0:1773941419.441053:0:12017:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a2fd240. 00000080:00000010:2.0:1773941419.441054:0:12017:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b85eec5e8. 00000080:00000010:2.0:1773941419.441056:0:12017:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b98a88920. 00000008:00000010:2.0:1773941419.441057:0:12017:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5b885b2dc0. 00020000:00000010:2.0:1773941419.441059:0:12017:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5b84cbfcc0. 00000080:00000010:2.0:1773941419.441064:0:12017:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b845f5d20. 00020000:00000010:2.0:1773941419.441069:0:12017:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffff8b5ba0166360. 00020000:00000010:2.0:1773941419.441074:0:12017:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffff8b5ba0166720. 00000020:00000010:2.0:1773941419.441077:0:12017:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b84feea00. 00000020:00000010:2.0:1773941419.441079:0:12017:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0166a20. 00000008:00000010:2.0:1773941419.441082:0:12017:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5ba0f04e18. 00020000:00000010:2.0:1773941419.441086:0:12017:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a2fd780. 00008000:00000010:2.0:1773941419.441088:0:12017:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d77880. 00000020:00000010:2.0:1773941419.441090:0:12017:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84fee600. 00000008:00000010:2.0:1773941419.441091:0:12017:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8b5b885b3e40. 00020000:00000010:2.0:1773941419.441093:0:12017:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5b84cbc240. 00008000:00000010:2.0:1773941419.441095:0:12017:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8b5b82c37248. 00000020:00000010:2.0:1773941419.441097:0:12017:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84fef200. 00000020:00000010:2.0:1773941419.441099:0:12017:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8b5b8509c2d0. 00000080:00000010:2.0:1773941419.441116:0:12017:0:(super25.c:43:ll_alloc_inode()) slab-alloced 'lli': 2072 at ffff8b5b83538840. 02000000:00000010:2.0:1773941419.441134:0:12017:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 131072 at ffff8b5b889a0000. 02000000:00000010:2.0:1773941419.441157:0:12017:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b84fef800. 00000100:00000010:2.0:1773941419.441162:0:12017:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021d500. 00000080:00000010:2.0:1773941419.441171:0:12017:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff8b5b98068140. 00000080:00000010:2.0:1773941419.441177:0:12017:0:(llite_lib.c:842:client_common_fill_super()) kfreed 'data': 192 at ffff8b5b885dee40. 00000080:00000010:2.0:1773941419.441181:0:12017:0:(llite_lib.c:843:client_common_fill_super()) kfreed 'osfs': 144 at ffff8b5b885ded80. 00000080:00000010:2.0:1773941419.441202:0:12017:0:(llite_lib.c:1521:ll_fill_super()) kfreed 'md': 33 at ffff8b5b850b07c0. 00000080:00000010:2.0:1773941419.441204:0:12017:0:(llite_lib.c:1523:ll_fill_super()) kfreed 'dt': 33 at ffff8b5b850b0f00. 00000080:00000010:2.0:1773941419.441207:0:12017:0:(llite_lib.c:1531:ll_fill_super()) kfreed 'cfg': 88 at ffff8b5ba0bc95a0. 00000080:02000400:2.0:1773941419.441209:0:12017:0:(llite_lib.c:1537:ll_fill_super()) Mounted lustre-client 00000400:00000001:2.0:1773941419.489273:0:12057:0:(debug.c:306:cfs_str2mask()) Process entered 00010000:00000001:1.0:1773941420.447768:0:255:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1773941420.447773:0:255:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941420.447813:0:12028:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941420.447816:0:12028:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941420.447818:0:12028:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:3.0:1773941420.640592:0:12064:0:(llite_lib.c:4092:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941420.754122:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697d08 00000080:00200008:1.0:1773941420.754130:0:12084:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b5b83538908), request_mask 2047, flags 0, foreign 0 00000080:00000001:1.0:1773941420.754138:0:12084:0:(file.c:6184:ll_inode_revalidate()) Process entered 00000020:00000001:1.0:1773941420.754140:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697c80 00000080:00200000:1.0:1773941420.754142:0:12084:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908),name=/ 00000080:00000010:1.0:1773941420.754147:0:12084:0:(llite_lib.c:3907:ll_prep_md_op_data()) kmalloced '(op_data)': 384 at ffff8b5b902a1c00. 00000080:00000001:1.0:1773941420.754153:0:12084:0:(namei.c:943:ll_intent_lock()) Process entered 00000020:00000010:1.0:1773941420.754158:0:12084:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffff8b5ba0735800. 00800000:00000001:1.0:1773941420.754162:0:12084:0:(lmv_intent.c:556:lmv_intent_lock()) Process entered 00000020:00000001:1.0:1773941420.754164:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697ba0 00800000:00000002:1.0:1773941420.754167:0:12084:0:(lmv_intent.c:563:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:1.0:1773941420.754171:0:12084:0:(lmv_intent.c:437:lmv_intent_lookup()) Process entered 00800000:00000002:1.0:1773941420.754174:0:12084:0:(lmv_intent.c:490:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000020:00000010:1.0:1773941420.754177:0:12084:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffff8b5ba0737800. 00000002:00000001:1.0:1773941420.754179:0:12084:0:(mdc_locks.c:1500:mdc_intent_lock()) Process entered 00000020:00000001:1.0:1773941420.754180:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697a70 00000002:00010000:1.0:1773941420.754182:0:12084:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:1.0:1773941420.754184:0:12084:0:(mdc_locks.c:1404:mdc_revalidate_lock()) Process entered 00000002:00000001:1.0:1773941420.754186:0:12084:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:1.0:1773941420.754188:0:12084:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:1.0:1773941420.754193:0:12084:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:1.0:1773941420.754194:0:12084:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:1.0:1773941420.754195:0:12084:0:(mdc_locks.c:1456:mdc_revalidate_lock()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:1.0:1773941420.754197:0:12084:0:(mdc_locks.c:1025:mdc_enqueue_base()) Process entered 00000002:00000001:1.0:1773941420.754199:0:12084:0:(mdc_locks.c:621:mdc_intent_getattr_pack()) Process entered 00000100:00000010:1.0:1773941420.754204:0:12084:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60000. 00000020:00000040:1.0:1773941420.754207:0:12084:0:(genops.c:1229:class_import_get()) import ffff8b5b88953800 refcount=4 obd=lustre-MDT0000-mdc-ffff8b5b885e5800 00010000:00000001:1.0:1773941420.754212:0:12084:0:(ldlm_request.c:853:ldlm_prep_elc_req()) Process entered 00010000:00000001:1.0:1773941420.754215:0:12084:0:(ldlm_request.c:1927:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1773941420.754218:0:12084:0:(ldlm_request.c:2099:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.754219:0:12084:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941420.754225:0:12084:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941420.754228:0:12084:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941420.754232:0:12084:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734400. 00000100:00000001:1.0:1773941420.754237:0:12084:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.754242:0:12084:0:(ldlm_request.c:2242:ldlm_cli_cancel_list()) Process entered 00010000:00000001:1.0:1773941420.754243:0:12084:0:(ldlm_request.c:2245:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.754244:0:12084:0:(ldlm_request.c:2242:ldlm_cli_cancel_list()) Process entered 00010000:00000001:1.0:1773941420.754245:0:12084:0:(ldlm_request.c:2245:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.754246:0:12084:0:(ldlm_request.c:909:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:1.0:1773941420.754249:0:12084:0:(mdc_locks.c:692:mdc_intent_getattr_pack()) Process leaving (rc=18446615824121069568 : -128249588482048 : ffff8b5b90d60000) 00010000:00000001:1.0:1773941420.754252:0:12084:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941420.754254:0:12084:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941420.754257:0:12084:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec400. 00010000:00000010:1.0:1773941420.754259:0:12084:0:(ldlm_resource.c:1452:ldlm_resource_inodebits_new()) kmalloced '(res->lr_ibits_queues)': 112 at ffff8b5b90edb600. 00010000:00000001:1.0:1773941420.754262:0:12084:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941420.754265:0:12084:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138ea00. 00000020:00000010:1.0:1773941420.754278:0:12084:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff8b5b90edba80. 00000020:00000001:1.0:1773941420.754281:0:12084:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941420.754283:0:12084:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138ea00 with handle 0xa33a5be14eae1c36 to hash 00000020:00000001:1.0:1773941420.754284:0:12084:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941420.754285:0:12084:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.754288:0:12084:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.754291:0:12084:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941420.754292:0:12084:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941420.754295:0:12084:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 12084 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941420.754302:0:12084:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 12084 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941420.754309:0:12084:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 12084 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941420.754312:0:12084:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941420.754317:0:12084:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941420.754321:0:12084:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b885def00. 00000100:00000001:1.0:1773941420.754322:0:12084:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615823978983168 : -128249730568448 : ffff8b5b885def00) 00000100:00000001:1.0:1773941420.754324:0:12084:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941420.754325:0:12084:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121069568 : -128249588482048 : ffff8b5b90d60000) 00000100:00000001:1.0:1773941420.754328:0:12084:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941420.754330:0:12084:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.754331:0:12084:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941420.754333:0:12084:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941420.754341:0:12084:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60000 x1860112241143040/t0(0) o101->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 576/1152 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:1.0:1773941420.754346:0:12084:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941420.754348:0:12084:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941420.754350:0:12084:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941420.754352:0:12084:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941420.754354:0:12084:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941420.754356:0:12084:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job lfs:637f784d-29a5-48a3-b831-1c4c7c7a2a54:12084:1860112241143040:192.168.204.143@tcp:101:lfs.0 00000100:00000001:1.0:1773941420.754358:0:12084:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941420.754362:0:12084:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941420.754364:0:12084:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941420.754366:0:12084:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941420.754370:0:12084:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff8b5b88968000. 02000000:00000001:1.0:1773941420.754371:0:12084:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941420.754376:0:12084:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941420.754379:0:12084:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034a50. 00000400:00000010:1.0:1773941420.754383:0:12084:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1088. 00000100:00000200:1.0:1773941420.754389:0:12084:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1860112241143040, portal 10 00000100:00000001:1.0:1773941420.754390:0:12084:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941420.754391:0:12084:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121069568 : -128249588482048 : ffff8b5b90d60000) 00000100:00000040:1.0:1773941420.754395:0:12084:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=600 req@ffff8b5b90d60000 x1860112241143040/t0(0) o101->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 576/1152 e 0 to 0 dl 1773941436 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:1.0:1773941420.754400:0:12084:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941420.754401:0:12084:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941420.754406:0:12084:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1cc0. 00000100:00000200:1.0:1773941420.754409:0:12084:0:(niobuf.c:94:ptl_send_buf()) Sending 576 bytes to portal 12, xid 1860112241143040, offset 0 00000400:00000200:1.0:1773941420.754415:0:12084:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941420.754422:0:12084:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941420.754429:0:12084:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941420.754433:0:12084:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941420.754437:0:12084:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941420.754441:0:12084:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941420.754444:0:12084:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941420.754447:0:12084:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [97:97:256:4294967295] 192.168.204.143@tcp LPNI seq info [97:97:8:4294967295] 00000400:00000200:1.0:1773941420.754452:0:12084:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941420.754462:0:12084:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941420.754467:0:12084:0:(socklnd_cb.c:1001:ksocknal_send()) sending 576 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941420.754472:0:12084:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024bc00. 00000800:00000200:1.0:1773941420.754479:0:12084:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941420.754486:0:12084:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941420.754491:0:12084:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024bc00 type 1, nob 672 niov 1 nkiov 1 00000100:00000001:1.0:1773941420.754505:0:12084:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941420.754509:0:12084:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b5b885e5800: using a timeout of 1837388 usecs (1838 jiffies) 00000400:00000200:1.0:1773941420.754513:0:12084:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941420.754515:0:12084:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b5b885e5800: PM QoS new deadline estimation for cpu 2 is 4294860384 00000100:00000001:1.0:1773941420.754521:0:12084:0:(niobuf.c:684:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b5b885e5800: PM QoS request active for cpu 2, simply extend its deadline from 4294859796 00000100:00000001:1.0:1773941420.754528:0:12084:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b5b885e5800: PM QoS new deadline estimation for cpu 3 is 4294860384 00000100:00000001:1.0:1773941420.754532:0:12084:0:(niobuf.c:684:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b5b885e5800: PM QoS request active for cpu 3, simply extend its deadline from 4294859796 00000100:00000001:1.0:1773941420.754535:0:12084:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941420.754537:0:12084:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.754538:0:12084:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941420.754540:0:12084:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941420.754542:0:12084:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b885def00 going to sleep for 11 seconds 00000100:00000001:1.0:1773941420.754544:0:12084:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941420.754550:0:12084:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941420.754625:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941420.754631:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024bc00. 00000400:00000200:3.0:1773941420.754635:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941420.754641:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941420.754645:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1cc0 00000400:00000010:3.0:1773941420.754648:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1cc0. 00000100:00000001:3.0:1773941420.754652:0:2346:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941420.754657:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60000 x1860112241143040/t0(0) o101->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 576/1152 e 0 to 0 dl 1773941436 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:3.0:1773941420.754677:0:2346:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941420.754681:0:2346:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d60000 x1860112241143040/t0(0) o101->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 576/1152 e 0 to 0 dl 1773941436 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:3.0:1773941420.754688:0:2346:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.754690:0:2346:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941420.756320:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.756331:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941420.756333:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.756335:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.756339:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.756344:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941420.756349:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.756351:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.756354:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.756357:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 688 into portal 10 MB=0x69bc321602d00 00000400:00000200:3.0:1773941420.756360:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.756364:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.143@tcp of length 688/688 into md 0x84d [1] + 224 00000800:00000001:3.0:1773941420.756368:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.756373:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.756374:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941420.756377:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941420.756380:0:2347:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941420.756384:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60000 x1860112241143040/t0(0) o101->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 576/1152 e 0 to 0 dl 1773941436 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000040:3.0:1773941420.756391:0:2347:0:(events.c:149:reply_in_callback()) @@@ reply in flags=600 mlen=688 offset=224 replen=1152 req@ffff8b5b90d60000 x1860112241143040/t0(0) o101->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 576/1152 e 0 to 0 dl 1773941436 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:3.0:1773941420.756403:0:2347:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941420.756408:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.756410:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941420.756436:0:12084:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941420.756443:0:12084:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941420.756444:0:12084:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941420.756455:0:12084:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034a50. 00000400:00000200:1.0:1773941420.756460:0:12084:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1088 00000400:00000010:1.0:1773941420.756462:0:12084:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1088. 00000100:00000001:1.0:1773941420.756465:0:12084:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941420.756470:0:12084:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60000 x1860112241143040/t0(0) o101->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 576/1152 e 0 to 0 dl 1773941436 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000200:1.0:1773941420.756480:0:12084:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60000 x1860112241143040/t0(0) o101->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 576/1152 e 0 to 0 dl 1773941436 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:1.0:1773941420.756486:0:12084:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941420.756488:0:12084:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941420.756491:0:12084:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941420.756494:0:12084:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941420.756496:0:12084:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941420.756498:0:12084:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941420.756500:0:12084:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941420.756508:0:12084:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b88953c58 time=1 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941420.756511:0:12084:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b88953c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941420.756514:0:12084:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941420.756515:0:12084:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.756518:0:12084:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941420.756522:0:12084:0:(ldlm_request.c:1566:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.756525:0:12084:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1773941420.756527:0:12084:0:(client.c:3077:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b5b885e5800: skip recheck: last_committed 0 00000100:00000001:1.0:1773941420.756529:0:12084:0:(client.c:3078:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1773941420.756531:0:12084:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941420.756535:0:12084:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60000 x1860112241143040/t0(0) o101->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 576/688 e 0 to 0 dl 1773941436 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:1.0:1773941420.756542:0:12084:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941420.756545:0:12084:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941420.756547:0:12084:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941420.756551:0:12084:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60000 x1860112241143040/t0(0) o101->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 576/688 e 0 to 0 dl 1773941436 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1773941420.756568:0:12084:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job lfs:637f784d-29a5-48a3-b831-1c4c7c7a2a54:12084:1860112241143040:192.168.204.143@tcp:101:lfs.0 00000100:00000001:1.0:1773941420.756573:0:12084:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941420.756575:0:12084:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.756577:0:12084:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941420.756579:0:12084:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941420.756582:0:12084:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60000 x1860112241143040/t0(0) o101->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 576/688 e 0 to 0 dl 1773941436 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:1.0:1773941420.756589:0:12084:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941420.756591:0:12084:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b885def00. 00000100:00000001:1.0:1773941420.756594:0:12084:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941420.756595:0:12084:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.756597:0:12084:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941420.756601:0:12084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941420.756603:0:12084:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941420.756606:0:12084:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=4 00000020:00000001:1.0:1773941420.756608:0:12084:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.756611:0:12084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000040:1.0:1773941420.756616:0:12084:0:(ldlm_request.c:689:ldlm_cli_enqueue_fini()) local: ffff8b5b8138ea00, remote cookie: 0xb574482a8fd85fac, flags: 0x1001 00010000:00010000:1.0:1773941420.756621:0:12084:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941420.756631:0:12084:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941420.756637:0:12084:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:1.0:1773941420.756641:0:12084:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:1.0:1773941420.756643:0:12084:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:1.0:1773941420.756644:0:12084:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:1.0:1773941420.756646:0:12084:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:1.0:1773941420.756648:0:12084:0:(ldlm_resource.c:1840:ldlm_resource_dump()) --- Resource: [0x200000007:0x1:0x0].0x0 (ffff8b5b890ec400) refcount = 1 00010000:00010000:1.0:1773941420.756652:0:12084:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941420.756663:0:12084:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00000020:00000010:1.0:1773941420.756668:0:12084:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 448 at ffff8b5b902a1400. 00010000:00000001:1.0:1773941420.756670:0:12084:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:1.0:1773941420.756672:0:12084:0:(ldlm_lock.c:1915:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941420.756675:0:12084:0:(ldlm_request.c:262:ldlm_completion_ast()) Process entered 00010000:00000001:1.0:1773941420.756677:0:12084:0:(ldlm_request.c:271:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941420.756680:0:12084:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941420.756686:0:12084:0:(ldlm_request.c:785:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:1.0:1773941420.756688:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.756689:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941420.756690:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.756691:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941420.756693:0:12084:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:1.0:1773941420.756697:0:12084:0:(mdc_locks.c:778:mdc_finish_enqueue()) Process entered 00010000:00000001:1.0:1773941420.756699:0:12084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941420.756700:0:12084:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941420.756702:0:12084:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=3 00000020:00000001:1.0:1773941420.756780:0:12084:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.756783:0:12084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.756785:0:12084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941420.756787:0:12084:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941420.756788:0:12084:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=4 00000020:00000001:1.0:1773941420.756790:0:12084:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.756792:0:12084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.756795:0:12084:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941420.756797:0:12084:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941420.756801:0:12084:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941420.756817:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.756819:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941420.756820:0:12084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941420.756821:0:12084:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941420.756823:0:12084:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=5 00000020:00000001:1.0:1773941420.756825:0:12084:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.756826:0:12084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.756829:0:12084:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941420.756831:0:12084:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941420.756837:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.756838:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941420.756841:0:12084:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941420.756847:0:12084:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941420.756848:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.756849:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941420.756850:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.756851:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:1.0:1773941420.756856:0:12084:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b5b90d60000 x1860112241143040/t0(0) o101->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 576/688 e 0 to 0 dl 1773941436 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00000001:1.0:1773941420.756864:0:12084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941420.756865:0:12084:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941420.756867:0:12084:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=3 00000020:00000001:1.0:1773941420.756868:0:12084:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.756870:0:12084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.756873:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.756874:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:1.0:1773941420.756875:0:12084:0:(mdc_locks.c:981:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:1.0:1773941420.756882:0:12084:0:(mdc_locks.c:1186:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:1.0:1773941420.756884:0:12084:0:(mdc_locks.c:1288:mdc_finish_intent_lock()) Process entered 00010000:00000001:1.0:1773941420.756887:0:12084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941420.756889:0:12084:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941420.756890:0:12084:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=3 00000020:00000001:1.0:1773941420.756892:0:12084:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.756894:0:12084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00000002:00010000:1.0:1773941420.756897:0:12084:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941420.756904:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.756905:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941420.756906:0:12084:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:1.0:1773941420.756907:0:12084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941420.756908:0:12084:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941420.756909:0:12084:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=3 00000020:00000001:1.0:1773941420.756911:0:12084:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.756913:0:12084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000040:1.0:1773941420.756917:0:12084:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8b5b890ec400 count: 2 00010000:00000001:1.0:1773941420.756920:0:12084:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:1.0:1773941420.756923:0:12084:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b5ba0c9e800 type 13 mode 4 res 8589934599/1 (0 0) 00010000:00000001:1.0:1773941420.756926:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.756927:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:1.0:1773941420.756928:0:12084:0:(mdc_locks.c:1383:mdc_finish_intent_lock()) Process leaving 00000020:00000001:1.0:1773941420.756930:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c416979f0 00000002:00002000:1.0:1773941420.756932:0:12084:0:(mdc_locks.c:1388:mdc_finish_intent_lock()) D_IT dentry= intent=getattr status=0 disp=b: rc = 0 00000002:00000001:1.0:1773941420.756935:0:12084:0:(mdc_locks.c:1543:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:1.0:1773941420.756938:0:12084:0:(lmv_intent.c:546:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:1.0:1773941420.756940:0:12084:0:(lmv_intent.c:596:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:1.0:1773941420.756943:0:12084:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00000001:1.0:1773941420.756946:0:12084:0:(namei.c:989:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:1.0:1773941420.756950:0:12084:0:(llite_lib.c:4048:ll_finish_md_op_data()) kfreed 'op_data': 384 at ffff8b5b902a1c00. 00000080:00000001:1.0:1773941420.756954:0:12084:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:1.0:1773941420.756956:0:12084:0:(llite_lib.c:3689:ll_prep_inode()) Process entered 00000002:00000001:1.0:1773941420.756965:0:12084:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00800000:00000001:1.0:1773941420.756970:0:12084:0:(lmv_obd.c:3775:lmv_stripe_object_create()) Process entered 00800000:00000010:1.0:1773941420.756974:0:12084:0:(lmv_obd.c:3754:lmv_stripe_object_alloc()) kmalloced '(lsm_obj)': 56 at ffff8b5b850b0640. 00800000:00000001:1.0:1773941420.756976:0:12084:0:(lmv_obd.c:3759:lmv_stripe_object_alloc()) Process leaving (rc=18446615823923217984 : -128249786333632 : ffff8b5b850b0640) 00800000:00000001:1.0:1773941420.756978:0:12084:0:(lmv_obd.c:3836:lmv_stripe_object_create()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:1.0:1773941420.756980:0:12084:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:1.0:1773941420.756983:0:12084:0:(llite_lib.c:1842:ll_update_lsm_md()) Process entered 00000080:00000002:1.0:1773941420.756985:0:12084:0:(llite_lib.c:1846:ll_update_lsm_md()) update lsm_obj 0000000000000000 of [0x200000007:0x1:0x0] 00000080:00000001:1.0:1773941420.756988:0:12084:0:(llite_lib.c:1797:ll_update_default_lsm_md()) Process entered 00000080:00000002:1.0:1773941420.756991:0:12084:0:(lustre_lmv.h:160:lmv_stripe_object_dump()) dump LMV: magic=0xcd30cd0 refs=1 count=1 index=4294967295 hash=invalid:0x0 max_inherit=0 max_inherit_rr=3 version=0 migrate_offset=0 migrate_hash=invalid:0 pool= 00000080:00000001:1.0:1773941420.756995:0:12084:0:(llite_lib.c:1832:ll_update_default_lsm_md()) Process leaving 00000080:00000001:1.0:1773941420.756997:0:12084:0:(llite_lib.c:1863:ll_update_lsm_md()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.757001:0:12084:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115188193296385 : 144115188193296385 : 200000007000001) 00000080:00000001:1.0:1773941420.757003:0:12084:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554432 : 33554432 : 2000000) 00000080:00200000:1.0:1773941420.757009:0:12084:0:(llite_lib.c:2979:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00010000:00000001:1.0:1773941420.757011:0:12084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941420.757013:0:12084:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941420.757014:0:12084:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=3 00000020:00000001:1.0:1773941420.757016:0:12084:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.757018:0:12084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.757020:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.757022:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:1.0:1773941420.757023:0:12084:0:(llite_lib.c:3787:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:1.0:1773941420.757026:0:12084:0:(lmv_obd.c:4012:lmv_put_lustre_md()) Process entered 00800000:00000001:1.0:1773941420.757028:0:12084:0:(lmv_obd.c:4019:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:1.0:1773941420.757031:0:12084:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) for lock 0xa33a5be14eae1c36 00800000:00000001:1.0:1773941420.757033:0:12084:0:(lmv_obd.c:3932:lmv_set_lock_data()) Process entered 00000002:00000001:1.0:1773941420.757035:0:12084:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:1.0:1773941420.757036:0:12084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941420.757037:0:12084:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941420.757038:0:12084:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=3 00000020:00000001:1.0:1773941420.757040:0:12084:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.757042:0:12084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.757050:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.757051:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:1.0:1773941420.757053:0:12084:0:(mdc_locks.c:122:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:1.0:1773941420.757055:0:12084:0:(lmv_obd.c:3937:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.757057:0:12084:0:(dcache.c:242:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:1.0:1773941420.757059:0:12084:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) 00000080:00010000:1.0:1773941420.757062:0:12084:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xa33a5be14eae1c36 from it ffffa10c41697cb8 00010000:00000001:1.0:1773941420.757064:0:12084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941420.757065:0:12084:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941420.757066:0:12084:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=3 00000020:00000001:1.0:1773941420.757068:0:12084:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.757071:0:12084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.757073:0:12084:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941420.757076:0:12084:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941420.757083:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.757084:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941420.757085:0:12084:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:1.0:1773941420.757087:0:12084:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:1.0:1773941420.757090:0:12084:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00000001:1.0:1773941420.757098:0:12084:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1773941420.757100:0:12084:0:(ldlm_request.c:2131:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1773941420.757101:0:12084:0:(ldlm_request.c:1927:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1773941420.757105:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.757106:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941420.757107:0:12084:0:(ldlm_request.c:2099:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.757110:0:12084:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1773941420.757112:0:12084:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.757114:0:12084:0:(ldlm_request.c:2141:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.757117:0:12084:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.757119:0:12084:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941420.757120:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.757125:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:1.0:1773941420.757127:0:12084:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941420.757131:0:12084:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60000 x1860112241143040/t0(0) o101->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 576/688 e 0 to 0 dl 1773941436 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:1.0:1773941420.757143:0:12084:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941420.757145:0:12084:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941420.757148:0:12084:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff8b5b88968000. 02000000:00000001:1.0:1773941420.757151:0:12084:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941420.757153:0:12084:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941420.757155:0:12084:0:(genops.c:1242:class_import_put()) import ffff8b5b88953800 refcount=3 obd=lustre-MDT0000-mdc-ffff8b5b885e5800 00000020:00000001:1.0:1773941420.757157:0:12084:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941420.757159:0:12084:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734400. 02000000:00000001:1.0:1773941420.757162:0:12084:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941420.757164:0:12084:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941420.757166:0:12084:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60000. 00000100:00000001:1.0:1773941420.757169:0:12084:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941420.757170:0:12084:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:1.0:1773941420.757174:0:12084:0:(file.c:6244:ll_merge_md_attr()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941420.757181:0:12084:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 1856 at ffff8b5b88968000. 00000020:00000001:1.0:1773941420.757185:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697d08 00000080:00000008:1.0:1773941420.757187:0:12084:0:(file.c:6469:ll_getattr_dentry()) COMPLETED file /:[0x200000007:0x1:0x0](ffff8b5b83538908), request_mask 2047, flags 0, foreign 0 00000080:00000001:1.0:1773941420.757631:0:12084:0:(llite_lib.c:4092:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941420.757650:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697d08 00000080:00200008:1.0:1773941420.757652:0:12084:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b5b83538908), request_mask 0, flags 0, foreign 0 00000080:00000001:1.0:1773941420.757654:0:12084:0:(file.c:6184:ll_inode_revalidate()) Process entered 00000020:00000001:1.0:1773941420.757655:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697c80 00000080:00200000:1.0:1773941420.757657:0:12084:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908),name=/ 00000080:00000010:1.0:1773941420.757660:0:12084:0:(llite_lib.c:3907:ll_prep_md_op_data()) kmalloced '(op_data)': 384 at ffff8b5b902a1c00. 00800000:00000002:1.0:1773941420.757663:0:12084:0:(lmv_obd.c:3847:lmv_stripe_object_get()) get ffff8b5b850b0640 2 00000080:00000001:1.0:1773941420.757665:0:12084:0:(namei.c:943:ll_intent_lock()) Process entered 00800000:00000001:1.0:1773941420.757667:0:12084:0:(lmv_intent.c:556:lmv_intent_lock()) Process entered 00000020:00000001:1.0:1773941420.757668:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697ba0 00800000:00000002:1.0:1773941420.757670:0:12084:0:(lmv_intent.c:563:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:1.0:1773941420.757672:0:12084:0:(lmv_intent.c:437:lmv_intent_lookup()) Process entered 00800000:00000002:1.0:1773941420.757674:0:12084:0:(lmv_intent.c:490:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:1.0:1773941420.757680:0:12084:0:(mdc_locks.c:1500:mdc_intent_lock()) Process entered 00000020:00000001:1.0:1773941420.757681:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697a70 00000002:00010000:1.0:1773941420.757683:0:12084:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:1.0:1773941420.757686:0:12084:0:(mdc_locks.c:1404:mdc_revalidate_lock()) Process entered 00000002:00000001:1.0:1773941420.757687:0:12084:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:1.0:1773941420.757689:0:12084:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:1.0:1773941420.757691:0:12084:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8b5b890ec400 count: 2 00010000:00000001:1.0:1773941420.757693:0:12084:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941420.757696:0:12084:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941420.757698:0:12084:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:1.0:1773941420.757701:0:12084:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 02000000:00000001:1.0:1773941420.757734:0:12084:0:(sec.c:1015:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:1.0:1773941420.757738:0:12084:0:(sec.c:1031:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941420.757741:0:12084:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00000001:1.0:1773941420.757748:0:12084:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:1.0:1773941420.757749:0:12084:0:(mdc_locks.c:1456:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:1.0:1773941420.757751:0:12084:0:(mdc_locks.c:1523:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:1.0:1773941420.757753:0:12084:0:(lmv_intent.c:511:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:1.0:1773941420.757755:0:12084:0:(lmv_intent.c:596:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:1.0:1773941420.757757:0:12084:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00000001:1.0:1773941420.757759:0:12084:0:(namei.c:989:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:1.0:1773941420.757761:0:12084:0:(lmv_obd.c:3892:lmv_stripe_object_put()) put ffff8b5b850b0640 2 00000080:00000010:1.0:1773941420.757763:0:12084:0:(llite_lib.c:4048:ll_finish_md_op_data()) kfreed 'op_data': 384 at ffff8b5b902a1c00. 00000080:00000001:1.0:1773941420.757766:0:12084:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:1.0:1773941420.757767:0:12084:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:1.0:1773941420.757769:0:12084:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) 00000080:00010000:1.0:1773941420.757772:0:12084:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) for lock 0xa33a5be14eae1c36 00800000:00000001:1.0:1773941420.757774:0:12084:0:(lmv_obd.c:3932:lmv_set_lock_data()) Process entered 00000002:00000001:1.0:1773941420.757776:0:12084:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:1.0:1773941420.757781:0:12084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941420.757782:0:12084:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941420.757783:0:12084:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=3 00000020:00000001:1.0:1773941420.757785:0:12084:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.757786:0:12084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.757788:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.757788:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:1.0:1773941420.757789:0:12084:0:(mdc_locks.c:122:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:1.0:1773941420.757790:0:12084:0:(lmv_obd.c:3937:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:1.0:1773941420.757791:0:12084:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xa33a5be14eae1c36 from it ffffa10c41697cb8 00010000:00000001:1.0:1773941420.757793:0:12084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941420.757793:0:12084:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941420.757794:0:12084:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=3 00000020:00000001:1.0:1773941420.757795:0:12084:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.757797:0:12084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.757798:0:12084:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941420.757799:0:12084:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00000001:1.0:1773941420.757803:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.757804:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941420.757804:0:12084:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:1.0:1773941420.757805:0:12084:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:1.0:1773941420.757807:0:12084:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00000001:1.0:1773941420.757810:0:12084:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1773941420.757811:0:12084:0:(ldlm_request.c:2131:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1773941420.757812:0:12084:0:(ldlm_request.c:1927:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1773941420.757813:0:12084:0:(ldlm_request.c:2099:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.757814:0:12084:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1773941420.757815:0:12084:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.757816:0:12084:0:(ldlm_request.c:2141:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.757817:0:12084:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.757820:0:12084:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941420.757821:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.757822:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:1.0:1773941420.757823:0:12084:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941420.757824:0:12084:0:(client.c:2844:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:1.0:1773941420.757826:0:12084:0:(file.c:6244:ll_merge_md_attr()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941420.757828:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697d08 00000080:00000008:1.0:1773941420.757829:0:12084:0:(file.c:6469:ll_getattr_dentry()) COMPLETED file /:[0x200000007:0x1:0x0](ffff8b5b83538908), request_mask 0, flags 0, foreign 0 00000080:00000001:1.0:1773941420.757889:0:12084:0:(file.c:6653:ll_inode_permission()) Process entered 00000080:00000001:1.0:1773941420.757891:0:12084:0:(file.c:6184:ll_inode_revalidate()) Process entered 00000020:00000001:1.0:1773941420.757892:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697b78 00000080:00200000:1.0:1773941420.757893:0:12084:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908),name=/ 00000080:00000010:1.0:1773941420.757895:0:12084:0:(llite_lib.c:3907:ll_prep_md_op_data()) kmalloced '(op_data)': 384 at ffff8b5b902a1c00. 00800000:00000002:1.0:1773941420.757897:0:12084:0:(lmv_obd.c:3847:lmv_stripe_object_get()) get ffff8b5b850b0640 2 00000080:00000001:1.0:1773941420.757898:0:12084:0:(namei.c:943:ll_intent_lock()) Process entered 00800000:00000001:1.0:1773941420.757900:0:12084:0:(lmv_intent.c:556:lmv_intent_lock()) Process entered 00000020:00000001:1.0:1773941420.757900:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697a98 00800000:00000002:1.0:1773941420.757902:0:12084:0:(lmv_intent.c:563:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:1.0:1773941420.757904:0:12084:0:(lmv_intent.c:437:lmv_intent_lookup()) Process entered 00800000:00000002:1.0:1773941420.757905:0:12084:0:(lmv_intent.c:490:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:1.0:1773941420.757907:0:12084:0:(mdc_locks.c:1500:mdc_intent_lock()) Process entered 00000020:00000001:1.0:1773941420.757908:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697968 00000002:00010000:1.0:1773941420.757909:0:12084:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:1.0:1773941420.757911:0:12084:0:(mdc_locks.c:1404:mdc_revalidate_lock()) Process entered 00000002:00000001:1.0:1773941420.757912:0:12084:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:1.0:1773941420.757913:0:12084:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:1.0:1773941420.757914:0:12084:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8b5b890ec400 count: 2 00010000:00000001:1.0:1773941420.757915:0:12084:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941420.757917:0:12084:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941420.757919:0:12084:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:1.0:1773941420.757921:0:12084:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 02000000:00000001:1.0:1773941420.757925:0:12084:0:(sec.c:1015:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:1.0:1773941420.757926:0:12084:0:(sec.c:1031:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941420.757931:0:12084:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000002:00000001:1.0:1773941420.757935:0:12084:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:1.0:1773941420.757936:0:12084:0:(mdc_locks.c:1456:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:1.0:1773941420.757937:0:12084:0:(mdc_locks.c:1523:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:1.0:1773941420.757938:0:12084:0:(lmv_intent.c:511:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:1.0:1773941420.757939:0:12084:0:(lmv_intent.c:596:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:1.0:1773941420.757940:0:12084:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00000001:1.0:1773941420.757942:0:12084:0:(namei.c:989:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:1.0:1773941420.757943:0:12084:0:(lmv_obd.c:3892:lmv_stripe_object_put()) put ffff8b5b850b0640 2 00000080:00000010:1.0:1773941420.757944:0:12084:0:(llite_lib.c:4048:ll_finish_md_op_data()) kfreed 'op_data': 384 at ffff8b5b902a1c00. 00000080:00000001:1.0:1773941420.757945:0:12084:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:1.0:1773941420.757946:0:12084:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:1.0:1773941420.757947:0:12084:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) 00000080:00010000:1.0:1773941420.757949:0:12084:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) for lock 0xa33a5be14eae1c36 00800000:00000001:1.0:1773941420.757950:0:12084:0:(lmv_obd.c:3932:lmv_set_lock_data()) Process entered 00000002:00000001:1.0:1773941420.757951:0:12084:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:1.0:1773941420.757952:0:12084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941420.757952:0:12084:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941420.757953:0:12084:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=3 00000020:00000001:1.0:1773941420.757955:0:12084:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.757956:0:12084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.757958:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.757958:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:1.0:1773941420.757959:0:12084:0:(mdc_locks.c:122:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:1.0:1773941420.757960:0:12084:0:(lmv_obd.c:3937:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:1.0:1773941420.757961:0:12084:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xa33a5be14eae1c36 from it ffffa10c41697bb0 00010000:00000001:1.0:1773941420.757962:0:12084:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941420.757962:0:12084:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941420.757963:0:12084:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=3 00000020:00000001:1.0:1773941420.757964:0:12084:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.757965:0:12084:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:1.0:1773941420.757966:0:12084:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941420.757970:0:12084:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00000001:1.0:1773941420.757974:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.757975:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941420.757976:0:12084:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:1.0:1773941420.757977:0:12084:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:1.0:1773941420.757978:0:12084:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00000001:1.0:1773941420.757981:0:12084:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1773941420.757982:0:12084:0:(ldlm_request.c:2131:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1773941420.757983:0:12084:0:(ldlm_request.c:1927:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1773941420.757984:0:12084:0:(ldlm_request.c:2099:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.757985:0:12084:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1773941420.757986:0:12084:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.757986:0:12084:0:(ldlm_request.c:2141:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.757988:0:12084:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.757989:0:12084:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941420.757989:0:12084:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941420.757990:0:12084:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:1.0:1773941420.757991:0:12084:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941420.757992:0:12084:0:(client.c:2844:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:1.0:1773941420.757993:0:12084:0:(file.c:6668:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908), inode mode 41ed mask 44 00000080:00000001:1.0:1773941420.757995:0:12084:0:(file.c:6707:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.757998:0:12084:0:(dir.c:2974:ll_dir_open()) Process entered 00000080:00000001:1.0:1773941420.758000:0:12084:0:(file.c:1021:ll_file_open()) Process entered 00000020:00000001:1.0:1773941420.758001:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697b78 00000080:00200008:1.0:1773941420.758002:0:12084:0:(file.c:1025:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b5b83538908), flags 100000 00000080:00000010:1.0:1773941420.758005:0:12084:0:(file.c:67:ll_file_data_get()) slab-alloced 'lfd': 288 at ffff8b5b817a7e00. 00000080:00000001:1.0:1773941420.758008:0:12084:0:(file.c:1050:ll_file_open()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.758009:0:12084:0:(dir.c:2975:ll_dir_open()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.758061:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.758063:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.758068:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.758071:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.758074:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.758080:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.758082:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.758084:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.758086:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.758090:0:12084:0:(obd_class.h:1067:obd_statfs()) Process entered 00800000:00000004:1.0:1773941420.758091:0:12084:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b5b885e5800: age 189, max_age 190 00000002:00000001:1.0:1773941420.758094:0:12084:0:(mdc_request.c:1658:mdc_statfs()) Process entered 00000020:00000040:1.0:1773941420.758095:0:12084:0:(genops.c:1229:class_import_get()) import ffff8b5b88953800 refcount=4 obd=lustre-MDT0000-mdc-ffff8b5b885e5800 00000100:00000010:1.0:1773941420.758098:0:12084:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60000. 00000020:00000040:1.0:1773941420.758100:0:12084:0:(genops.c:1229:class_import_get()) import ffff8b5b88953800 refcount=5 obd=lustre-MDT0000-mdc-ffff8b5b885e5800 00000100:00000001:1.0:1773941420.758103:0:12084:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941420.758104:0:12084:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941420.758106:0:12084:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941420.758110:0:12084:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b9024b900. 00000100:00000001:1.0:1773941420.758114:0:12084:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.758117:0:12084:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941420.758119:0:12084:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941420.758122:0:12084:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b885def00. 00000100:00000001:1.0:1773941420.758124:0:12084:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615823978983168 : -128249730568448 : ffff8b5b885def00) 00000100:00000001:1.0:1773941420.758127:0:12084:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941420.758128:0:12084:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121069568 : -128249588482048 : ffff8b5b90d60000) 00000100:00000001:1.0:1773941420.758132:0:12084:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941420.758135:0:12084:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.758137:0:12084:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941420.758138:0:12084:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941420.758143:0:12084:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60000 x1860112241143168/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941420.758150:0:12084:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941420.758152:0:12084:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941420.758156:0:12084:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941420.758158:0:12084:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941420.758160:0:12084:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941420.758163:0:12084:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job lfs:637f784d-29a5-48a3-b831-1c4c7c7a2a54:12084:1860112241143168:192.168.204.143@tcp:41:lfs.0 00000100:00000001:1.0:1773941420.758167:0:12084:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941420.758176:0:12084:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941420.758178:0:12084:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941420.758181:0:12084:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941420.758185:0:12084:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735000. 02000000:00000001:1.0:1773941420.758187:0:12084:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941420.758192:0:12084:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941420.758196:0:12084:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034a50. 00000400:00000010:1.0:1773941420.758201:0:12084:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1088. 00000100:00000200:1.0:1773941420.758206:0:12084:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241143168, portal 10 00000100:00000001:1.0:1773941420.758208:0:12084:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941420.758210:0:12084:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121069568 : -128249588482048 : ffff8b5b90d60000) 00000100:00000040:1.0:1773941420.758216:0:12084:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60000 x1860112241143168/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941436 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941420.758223:0:12084:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941420.758226:0:12084:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941420.758228:0:12084:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1dd0. 00000100:00000200:1.0:1773941420.758232:0:12084:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1860112241143168, offset 0 00000400:00000200:1.0:1773941420.758237:0:12084:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941420.758243:0:12084:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941420.758248:0:12084:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941420.758252:0:12084:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941420.758256:0:12084:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941420.758258:0:12084:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941420.758261:0:12084:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941420.758264:0:12084:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [98:98:256:4294967295] 192.168.204.143@tcp LPNI seq info [98:98:8:4294967295] 00000400:00000200:1.0:1773941420.758282:0:12084:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941420.758290:0:12084:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941420.758296:0:12084:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941420.758300:0:12084:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b000. 00000800:00000200:1.0:1773941420.758306:0:12084:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941420.758312:0:12084:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941420.758316:0:12084:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b000 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941420.758334:0:12084:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941420.758338:0:12084:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b5b885e5800: using a timeout of 1608053 usecs (1609 jiffies) 00000400:00000200:1.0:1773941420.758343:0:12084:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941420.758345:0:12084:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b5b885e5800: PM QoS new deadline estimation for cpu 2 is 4294860159 00000100:00000001:1.0:1773941420.758349:0:12084:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b5b885e5800: PM QoS request active for cpu 2, keep current deadline 4294860384 00000100:00000001:1.0:1773941420.758351:0:12084:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b5b885e5800: PM QoS new deadline estimation for cpu 3 is 4294860159 00000100:00000001:1.0:1773941420.758354:0:12084:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b5b885e5800: PM QoS request active for cpu 3, keep current deadline 4294860384 00000100:00000001:1.0:1773941420.758357:0:12084:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941420.758359:0:12084:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.758361:0:12084:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941420.758363:0:12084:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941420.758366:0:12084:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b885def00 going to sleep for 11 seconds 00000100:00000001:1.0:1773941420.758369:0:12084:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941420.758371:0:12084:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941420.758384:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941420.758389:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b000. 00000400:00000200:3.0:1773941420.758398:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941420.758403:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941420.758408:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1dd0 00000400:00000010:3.0:1773941420.758410:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1dd0. 00000100:00000001:3.0:1773941420.758414:0:2346:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941420.758419:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60000 x1860112241143168/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941436 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.758428:0:2346:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941420.758432:0:2346:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d60000 x1860112241143168/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941436 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.758440:0:2346:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.758442:0:2346:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941420.759309:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.759314:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941420.759316:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.759317:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.759321:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.759324:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941420.759328:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.759330:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.759332:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.759335:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 368 into portal 10 MB=0x69bc321602d80 00000400:00000200:3.0:1773941420.759337:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.759340:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.143@tcp of length 368/368 into md 0x855 [1] + 224 00000800:00000001:3.0:1773941420.759344:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.759347:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.759348:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941420.759350:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941420.759353:0:2347:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941420.759356:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60000 x1860112241143168/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941436 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941420.759363:0:2347:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=368 offset=224 replen=368 req@ffff8b5b90d60000 x1860112241143168/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941436 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.759380:0:2347:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941420.759384:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.759386:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941420.759403:0:12084:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941420.759407:0:12084:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941420.759408:0:12084:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941420.759409:0:12084:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034a50. 00000400:00000200:1.0:1773941420.759412:0:12084:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1088 00000400:00000010:1.0:1773941420.759414:0:12084:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1088. 00000100:00000001:1.0:1773941420.759415:0:12084:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941420.759419:0:12084:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60000 x1860112241143168/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941436 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941420.759425:0:12084:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60000 x1860112241143168/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941436 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941420.759430:0:12084:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941420.759430:0:12084:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941420.759432:0:12084:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941420.759434:0:12084:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941420.759435:0:12084:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941420.759436:0:12084:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941420.759438:0:12084:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941420.759442:0:12084:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b88953c58 time=1 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941420.759448:0:12084:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b88953c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941420.759450:0:12084:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941420.759451:0:12084:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.759453:0:12084:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941420.759454:0:12084:0:(ldlm_request.c:1546:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.759456:0:12084:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1773941420.759457:0:12084:0:(client.c:3077:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b5b885e5800: skip recheck: last_committed 0 00000100:00000001:1.0:1773941420.759459:0:12084:0:(client.c:3078:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1773941420.759459:0:12084:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941420.759462:0:12084:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60000 x1860112241143168/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941436 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941420.759466:0:12084:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941420.759468:0:12084:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941420.759469:0:12084:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941420.759471:0:12084:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60000 x1860112241143168/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941436 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941420.759475:0:12084:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job lfs:637f784d-29a5-48a3-b831-1c4c7c7a2a54:12084:1860112241143168:192.168.204.143@tcp:41:lfs.0 00000100:00000001:1.0:1773941420.759478:0:12084:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941420.759479:0:12084:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.759480:0:12084:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941420.759481:0:12084:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941420.759483:0:12084:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60000 x1860112241143168/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941436 ref 2 fl Complete:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941420.759486:0:12084:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941420.759487:0:12084:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b885def00. 00000100:00000001:1.0:1773941420.759489:0:12084:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941420.759490:0:12084:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:1.0:1773941420.759493:0:12084:0:(mdc_request.c:1710:mdc_statfs()) Process leaving 00000100:00000001:1.0:1773941420.759494:0:12084:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941420.759496:0:12084:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60000 x1860112241143168/t0(0) o41->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941436 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941420.759500:0:12084:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941420.759503:0:12084:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941420.759505:0:12084:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735000. 02000000:00000001:1.0:1773941420.759507:0:12084:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941420.759508:0:12084:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941420.759509:0:12084:0:(genops.c:1242:class_import_put()) import ffff8b5b88953800 refcount=4 obd=lustre-MDT0000-mdc-ffff8b5b885e5800 00000020:00000001:1.0:1773941420.759510:0:12084:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941420.759511:0:12084:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b9024b900. 02000000:00000001:1.0:1773941420.759513:0:12084:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941420.759514:0:12084:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941420.759515:0:12084:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60000. 00000100:00000001:1.0:1773941420.759516:0:12084:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941420.759517:0:12084:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1773941420.759518:0:12084:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941420.759519:0:12084:0:(genops.c:1242:class_import_put()) import ffff8b5b88953800 refcount=3 obd=lustre-MDT0000-mdc-ffff8b5b885e5800 00000020:00000001:1.0:1773941420.759520:0:12084:0:(genops.c:1249:class_import_put()) Process leaving 00800000:00000004:1.0:1773941420.759521:0:12084:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b5b885e5800: update ffff8b5b85511e08 cache blocks 20248/23812 objects 79720/79992 00800000:00000001:1.0:1773941420.759524:0:12084:0:(obd_class.h:1128:obd_statfs()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:1.0:1773941420.759526:0:12084:0:(lmv_obd.c:1164:lmv_iocontrol()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759527:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:1.0:1773941420.759529:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759531:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759532:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759559:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759560:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759564:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759565:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759567:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759569:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759571:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759572:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759573:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759575:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759577:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759579:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759580:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759582:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759586:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759589:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759590:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759592:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759593:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759595:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759596:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759597:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759598:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759599:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759600:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759601:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759602:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759604:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759605:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759606:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759608:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759609:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759610:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759611:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759613:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759614:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759615:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759616:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759616:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759618:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759619:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759620:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759621:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759623:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759624:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759631:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759632:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759638:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759639:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759641:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759642:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759643:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759644:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759645:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759646:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759648:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759649:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759650:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759651:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759652:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759655:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759656:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759657:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759658:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759659:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759660:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759661:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759662:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759663:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759664:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759666:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759667:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759668:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759670:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759671:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759673:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759674:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759675:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759676:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759677:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759681:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759682:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759683:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759684:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759685:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759686:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759688:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759689:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759690:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759691:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759693:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759694:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759696:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759697:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759698:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759699:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759700:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759701:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759719:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759721:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759723:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759724:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759726:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759728:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759729:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759731:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759732:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759734:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759734:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759736:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759738:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759739:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759740:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759744:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759746:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759747:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759749:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759750:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759752:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759753:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759755:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759756:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759757:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759758:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759760:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759761:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759762:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759763:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759764:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759765:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759767:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759768:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759770:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759771:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759772:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759774:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759775:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759776:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759777:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759778:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759779:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759779:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759780:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759781:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759782:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759783:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759786:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759787:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759789:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759790:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759792:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759793:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759794:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759795:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759796:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759797:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759798:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759798:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759799:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759800:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759801:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759802:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759803:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759804:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759805:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759807:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759807:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759808:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759809:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759810:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759811:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759812:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759812:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759813:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759814:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759815:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759816:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759817:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759821:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759822:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759824:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759824:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759826:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759826:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759827:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759829:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759829:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759830:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759831:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759832:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759833:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759834:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759835:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759836:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759837:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759838:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759839:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759840:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759841:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759842:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759843:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759843:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759844:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759845:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759846:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759847:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759848:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759849:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759850:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759851:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759852:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759855:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759857:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759858:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759859:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759860:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759861:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759861:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759862:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759863:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759864:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759865:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759866:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759867:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759868:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759870:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759871:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759872:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759872:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759873:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759874:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759875:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759876:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759876:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759878:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759879:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759880:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759881:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759882:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759883:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759884:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759885:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759886:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759886:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759890:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759891:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759892:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759892:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759893:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759894:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759895:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759896:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759897:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759898:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759899:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759901:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759902:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759904:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759904:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759905:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759906:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759907:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759907:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759908:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759909:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759910:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759911:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759912:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759913:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759914:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759916:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759916:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759917:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759918:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759919:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759920:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759921:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759925:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759925:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759927:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759928:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759929:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759930:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759931:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759932:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759934:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759935:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759936:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759937:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759938:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759939:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759940:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759940:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759941:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759942:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759943:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759944:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759945:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759946:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759947:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759948:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759949:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759950:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759951:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759952:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759953:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759953:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759954:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759955:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759956:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759959:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759960:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759962:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759963:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759964:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759966:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759967:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759968:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759969:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759970:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759971:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759972:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759972:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759973:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759974:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759980:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759981:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.759982:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.759984:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759985:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759986:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.759987:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.759990:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.759991:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.759992:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.759993:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.759994:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.759995:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.759996:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.759997:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.759998:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760000:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760001:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760005:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760006:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760008:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760009:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760011:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760012:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760013:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760014:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760015:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760015:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760016:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760017:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760018:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760019:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760020:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760021:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760022:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760024:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760024:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760025:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760026:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760027:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760028:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760029:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760029:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760030:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760031:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760032:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760033:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760034:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760035:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760036:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760037:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760041:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760043:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760044:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760046:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760048:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760050:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760051:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760052:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760055:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760057:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760058:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760061:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760063:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760065:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760069:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760071:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760074:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760075:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760077:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760079:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760081:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760082:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760083:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760086:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760088:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760090:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760093:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760095:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760096:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760099:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760101:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760104:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760105:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760111:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760113:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760115:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760116:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760118:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760120:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760122:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760124:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760126:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760128:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760129:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760132:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760133:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760135:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760136:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760137:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760138:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760139:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760140:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760141:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760142:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760143:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760144:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760145:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760146:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760147:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760149:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760149:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760151:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760151:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760152:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760153:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760154:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760157:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760158:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760160:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760161:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760162:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760163:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760164:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760165:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760167:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760168:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760169:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760170:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760171:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760172:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760173:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760173:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760174:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760175:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760176:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760177:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760178:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760179:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760180:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760181:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760182:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760183:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760184:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760185:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760186:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760187:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760187:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760188:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760189:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760193:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760194:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760195:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760196:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760197:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760200:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760200:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760202:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760202:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760203:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760204:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760205:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760206:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760207:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760208:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760209:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760210:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760211:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760212:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760213:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760214:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760215:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760216:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760216:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760217:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760218:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760219:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760220:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760220:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760222:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760222:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760223:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760224:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760229:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760230:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760232:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760232:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760234:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760235:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760236:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760237:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760237:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760238:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760239:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760240:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760241:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760242:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760243:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760244:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760245:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760246:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760247:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760248:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760249:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760250:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760251:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760251:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760252:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760253:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760254:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760255:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760256:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760257:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760258:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760259:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760260:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760263:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760265:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760265:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760278:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760281:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760282:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760283:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760283:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760285:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760286:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760287:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760288:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760290:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760291:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760294:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760295:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760296:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760297:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760298:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760299:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760300:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760301:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760302:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760303:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760304:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760305:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760306:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760307:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760308:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760309:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760310:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760311:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760312:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760316:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760317:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760318:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760319:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760319:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760321:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760322:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760323:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760324:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760325:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760326:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760328:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760329:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760330:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760331:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760332:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760333:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760334:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760334:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760335:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760336:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760337:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760338:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760339:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760340:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760341:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760342:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760343:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760344:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760345:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760345:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760346:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760347:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760350:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760351:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760353:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760354:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760355:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760356:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760357:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760358:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760360:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760361:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760362:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760363:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760364:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760365:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760366:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760366:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760367:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760368:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760369:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760370:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760371:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760372:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760373:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760374:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760375:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760376:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760377:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760377:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760379:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760379:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760380:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760380:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760382:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760385:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760386:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760387:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760388:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760389:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760391:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760392:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760393:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760394:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760395:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760396:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760397:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760397:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760398:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760399:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760400:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760401:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760402:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760403:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760404:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760405:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760406:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760407:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760408:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760409:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760410:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760410:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760411:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760412:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760413:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760414:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760415:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760416:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760419:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760420:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760422:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760423:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760424:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760425:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760426:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760427:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760428:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760428:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760429:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760430:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760431:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760432:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760433:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760434:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760435:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760437:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760437:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760438:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760439:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760440:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760441:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760442:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760442:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760443:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760444:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760445:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760446:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760447:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760448:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760449:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760450:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760453:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760455:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760456:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760457:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760458:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760458:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760459:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760460:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760461:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760462:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760463:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760464:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760465:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760466:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760468:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760469:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760470:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760470:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760471:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760472:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760473:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760474:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760474:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760475:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760476:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760477:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760478:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760479:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760480:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760482:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760482:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760483:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760484:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760488:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760489:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760490:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760491:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760491:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760493:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760494:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760495:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760496:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760497:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760498:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760500:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760500:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760502:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760502:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760503:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760504:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760505:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760506:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760506:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760507:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760508:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760509:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760510:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760511:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760512:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760514:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760514:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760515:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760516:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760517:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760518:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760519:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760522:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760522:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760524:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760525:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760526:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760527:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760528:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760529:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760531:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760532:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760533:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760534:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760535:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760536:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760537:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760537:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760538:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760539:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760540:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760541:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760542:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760543:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760544:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760545:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760546:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760547:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760548:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760549:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760550:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760550:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760551:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760552:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760553:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760556:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760557:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760558:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760559:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760560:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760562:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760563:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760564:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760565:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760566:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760567:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760568:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760568:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760569:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760570:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760571:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760572:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760573:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760574:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760575:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760577:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760577:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760578:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760579:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760580:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760581:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760582:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760582:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760583:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760584:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760585:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760586:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760587:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760590:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760591:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760593:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760594:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760595:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760596:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760597:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760598:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760599:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760600:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760600:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760601:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760602:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760603:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760604:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760605:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760606:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760608:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760608:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760610:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760610:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760611:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760612:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760613:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760614:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760614:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760615:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760616:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760617:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760618:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760619:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760620:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760622:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760625:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760626:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760627:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760628:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760629:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760630:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760630:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760631:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760632:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760633:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760634:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760635:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760636:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760637:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760639:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760640:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760641:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760642:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760643:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760644:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760644:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760645:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760646:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760647:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760648:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760649:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760650:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760651:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760652:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760653:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760654:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760655:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760655:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760659:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760660:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760661:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760661:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760662:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760663:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760664:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760665:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760666:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760667:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760668:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760670:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760671:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760672:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760673:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760674:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760675:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760675:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760676:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760677:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760678:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760679:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760680:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760681:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760682:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760683:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760684:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760685:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760686:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760686:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760687:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760688:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760689:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760692:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760693:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760694:0:12084:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760695:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760696:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.760697:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760698:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760699:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.760701:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760702:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760722:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760723:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760725:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760726:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760728:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:1.0:1773941420.760728:0:12084:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:1.0:1773941420.760729:0:12084:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00800000:00000001:1.0:1773941420.760730:0:12084:0:(lmv_obd.c:935:lmv_iocontrol()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:1.0:1773941420.760732:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:1.0:1773941420.760733:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551597 : -19 : 0xffffffffffffffed) 00000080:00000010:1.0:1773941420.760734:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.760735:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:1.0:1773941420.760737:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:1.0:1773941420.760739:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.760740:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.760741:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.760742:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.760744:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.760745:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.760746:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.760748:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.760749:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.760753:0:12084:0:(obd_class.h:1067:obd_statfs()) Process entered 00020000:00000004:1.0:1773941420.760758:0:12084:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b5b885e5800: age -816, max_age 190 00000008:00000001:1.0:1773941420.760760:0:12084:0:(osc_request.c:3449:osc_statfs()) Process entered 00000020:00000040:1.0:1773941420.760762:0:12084:0:(genops.c:1229:class_import_get()) import ffff8b5b88957800 refcount=3 obd=lustre-OST0000-osc-ffff8b5b885e5800 00000100:00000010:1.0:1773941420.760765:0:12084:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60000. 00000020:00000040:1.0:1773941420.760766:0:12084:0:(genops.c:1229:class_import_get()) import ffff8b5b88957800 refcount=4 obd=lustre-OST0000-osc-ffff8b5b885e5800 00000020:00000001:1.0:1773941420.760768:0:12084:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941420.760769:0:12084:0:(genops.c:1242:class_import_put()) import ffff8b5b88957800 refcount=3 obd=lustre-OST0000-osc-ffff8b5b885e5800 00000020:00000001:1.0:1773941420.760770:0:12084:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941420.760771:0:12084:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941420.760773:0:12084:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941420.760775:0:12084:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941420.760777:0:12084:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b9024b900. 00000100:00000001:1.0:1773941420.760780:0:12084:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.760782:0:12084:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941420.760783:0:12084:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941420.760785:0:12084:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b885def00. 00000100:00000001:1.0:1773941420.760786:0:12084:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615823978983168 : -128249730568448 : ffff8b5b885def00) 00000100:00000001:1.0:1773941420.760787:0:12084:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941420.760788:0:12084:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121069568 : -128249588482048 : ffff8b5b90d60000) 00000100:00000001:1.0:1773941420.760790:0:12084:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941420.760792:0:12084:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.760793:0:12084:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941420.760794:0:12084:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941420.760797:0:12084:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60000 x1860112241143296/t0(0) o13->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941420.760802:0:12084:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941420.760803:0:12084:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941420.760804:0:12084:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941420.760806:0:12084:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941420.760807:0:12084:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941420.760809:0:12084:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job lfs:637f784d-29a5-48a3-b831-1c4c7c7a2a54:12084:1860112241143296:192.168.204.143@tcp:13:lfs.0 00000100:00000001:1.0:1773941420.760811:0:12084:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941420.760813:0:12084:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941420.760814:0:12084:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941420.760818:0:12084:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941420.760819:0:12084:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735000. 02000000:00000001:1.0:1773941420.760821:0:12084:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941420.760824:0:12084:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941420.760825:0:12084:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034a50. 00000400:00000010:1.0:1773941420.760828:0:12084:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1088. 00000100:00000200:1.0:1773941420.760830:0:12084:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241143296, portal 4 00000100:00000001:1.0:1773941420.760832:0:12084:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941420.760832:0:12084:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121069568 : -128249588482048 : ffff8b5b90d60000) 00000100:00000040:1.0:1773941420.760836:0:12084:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60000 x1860112241143296/t0(0) o13->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941436 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941420.760841:0:12084:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941420.760842:0:12084:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941420.760844:0:12084:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941420.760845:0:12084:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1860112241143296, offset 0 00000400:00000200:1.0:1773941420.760848:0:12084:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941420.760850:0:12084:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941420.760853:0:12084:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941420.760855:0:12084:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941420.760857:0:12084:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941420.760858:0:12084:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941420.760860:0:12084:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941420.760862:0:12084:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [99:99:256:4294967295] 192.168.204.143@tcp LPNI seq info [99:99:8:4294967295] 00000400:00000200:1.0:1773941420.760865:0:12084:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941420.760869:0:12084:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941420.760871:0:12084:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941420.760874:0:12084:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b000. 00000800:00000200:1.0:1773941420.760877:0:12084:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941420.760881:0:12084:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941420.760883:0:12084:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b000 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941420.760891:0:12084:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941420.760897:0:12084:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff8b5b885e5800: using a timeout of 2678 usecs (3 jiffies) 00000400:00000200:1.0:1773941420.760899:0:12084:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941420.760901:0:12084:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff8b5b885e5800: PM QoS new deadline estimation for cpu 2 is 4294858556 00000100:00000001:1.0:1773941420.760903:0:12084:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff8b5b885e5800: PM QoS request active for cpu 2, keep current deadline 4294860384 00000100:00000001:1.0:1773941420.760905:0:12084:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff8b5b885e5800: PM QoS new deadline estimation for cpu 3 is 4294858556 00000100:00000001:1.0:1773941420.760906:0:12084:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff8b5b885e5800: PM QoS request active for cpu 3, keep current deadline 4294860384 00000100:00000001:1.0:1773941420.760908:0:12084:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941420.760909:0:12084:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.760911:0:12084:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941420.760912:0:12084:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941420.760913:0:12084:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b885def00 going to sleep for 11 seconds 00000100:00000001:1.0:1773941420.760915:0:12084:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941420.760916:0:12084:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941420.760934:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941420.760937:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b000. 00000400:00000200:3.0:1773941420.760939:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941420.760942:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941420.760945:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:3.0:1773941420.760946:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:3.0:1773941420.760948:0:2346:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941420.760951:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60000 x1860112241143296/t0(0) o13->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941436 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.760956:0:2346:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941420.760958:0:2346:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d60000 x1860112241143296/t0(0) o13->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941436 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.760962:0:2346:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.760963:0:2346:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941420.761848:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.761856:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941420.761858:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.761861:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.761865:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.761869:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941420.761876:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.761878:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.761882:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.761885:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 368 into portal 4 MB=0x69bc321602e00 00000400:00000200:3.0:1773941420.761889:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.761893:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.143@tcp of length 368/368 into md 0x85d [1] + 224 00000800:00000001:3.0:1773941420.761897:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.761903:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.761906:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941420.761915:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941420.761920:0:2347:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941420.761924:0:2347:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60000 x1860112241143296/t0(0) o13->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941436 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941420.761934:0:2347:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=368 offset=224 replen=368 req@ffff8b5b90d60000 x1860112241143296/t0(0) o13->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941436 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.761949:0:2347:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941420.761954:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.761957:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941420.761964:0:12084:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941420.761968:0:12084:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941420.761969:0:12084:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941420.761971:0:12084:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034a50. 00000400:00000200:1.0:1773941420.761973:0:12084:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1088 00000400:00000010:1.0:1773941420.761974:0:12084:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1088. 00000100:00000001:1.0:1773941420.761976:0:12084:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941420.761979:0:12084:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60000 x1860112241143296/t0(0) o13->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941436 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941420.761986:0:12084:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60000 x1860112241143296/t0(0) o13->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941436 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941420.761990:0:12084:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941420.761991:0:12084:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941420.761992:0:12084:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941420.761993:0:12084:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941420.761994:0:12084:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941420.761995:0:12084:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941420.761997:0:12084:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941420.762000:0:12084:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b88957ca0 time=1773941420 v=5 (0 0 0 0) 00000100:00001000:1.0:1773941420.762002:0:12084:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b88957c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941420.762008:0:12084:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941420.762009:0:12084:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.762011:0:12084:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1773941420.762014:0:12084:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324) req@ffff8b5b90d60000 x1860112241143296/t0(0) o13->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941436 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941420.762018:0:12084:0:(ldlm_request.c:1535:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.762020:0:12084:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1773941420.762021:0:12084:0:(client.c:3077:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b5b885e5800: skip recheck: last_committed 0 00000100:00000001:1.0:1773941420.762022:0:12084:0:(client.c:3078:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1773941420.762023:0:12084:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941420.762025:0:12084:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60000 x1860112241143296/t0(0) o13->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941436 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941420.762029:0:12084:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941420.762030:0:12084:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941420.762031:0:12084:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941420.762032:0:12084:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60000 x1860112241143296/t0(0) o13->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941436 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941420.762037:0:12084:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job lfs:637f784d-29a5-48a3-b831-1c4c7c7a2a54:12084:1860112241143296:192.168.204.143@tcp:13:lfs.0 00000100:00000001:1.0:1773941420.762039:0:12084:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941420.762040:0:12084:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.762042:0:12084:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941420.762043:0:12084:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941420.762044:0:12084:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60000 x1860112241143296/t0(0) o13->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941436 ref 2 fl Complete:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941420.762048:0:12084:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941420.762049:0:12084:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b885def00. 00000100:00000001:1.0:1773941420.762051:0:12084:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941420.762051:0:12084:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1773941420.762054:0:12084:0:(osc_request.c:3497:osc_statfs()) Process leaving 00000100:00000001:1.0:1773941420.762055:0:12084:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941420.762056:0:12084:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60000 x1860112241143296/t0(0) o13->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941436 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941420.762066:0:12084:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941420.762067:0:12084:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941420.762069:0:12084:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735000. 02000000:00000001:1.0:1773941420.762070:0:12084:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941420.762071:0:12084:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941420.762072:0:12084:0:(genops.c:1242:class_import_put()) import ffff8b5b88957800 refcount=2 obd=lustre-OST0000-osc-ffff8b5b885e5800 00000020:00000001:1.0:1773941420.762073:0:12084:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941420.762075:0:12084:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b9024b900. 02000000:00000001:1.0:1773941420.762076:0:12084:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941420.762077:0:12084:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941420.762078:0:12084:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60000. 00000100:00000001:1.0:1773941420.762080:0:12084:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941420.762080:0:12084:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00020000:00000004:1.0:1773941420.762082:0:12084:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b5b885e5800: update ffff8b5ba0739e08 cache blocks 31706/35554 objects 49666/50000 00020000:00000001:1.0:1773941420.762084:0:12084:0:(obd_class.h:1128:obd_statfs()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:1.0:1773941420.762087:0:12084:0:(lov_obd.c:1128:lov_iocontrol()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762088:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:1.0:1773941420.762089:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762090:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762091:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762118:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762120:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762123:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762124:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762126:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762128:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762129:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762130:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762131:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762133:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762135:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762136:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762138:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762139:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762143:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762146:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762147:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762149:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762150:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762151:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762152:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762153:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762154:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762155:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762156:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762158:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762159:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762160:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762161:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762162:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762164:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762165:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762166:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762167:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762168:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762169:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762170:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762171:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762171:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762173:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762174:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762175:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762176:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762177:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762178:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762180:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762181:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762185:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762186:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762187:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762188:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762189:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762190:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762191:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762192:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762193:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762194:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762195:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762196:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762197:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762199:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762200:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762201:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762202:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762204:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762205:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762206:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762207:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762207:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762208:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762210:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762211:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762212:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762213:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762214:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762215:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762216:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762218:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762218:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762220:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762225:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762226:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762227:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762228:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762229:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762230:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762231:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762233:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762238:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762240:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762242:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762243:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762245:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762246:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762248:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762249:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762250:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762250:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762251:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762253:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762254:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762256:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762257:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762258:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762259:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762261:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762262:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762264:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762264:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762276:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762278:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762279:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762280:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762285:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762287:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762289:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762290:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762292:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762294:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762296:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762299:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762300:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762301:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762302:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762303:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762305:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762306:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762306:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762307:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762308:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762310:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762311:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762313:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762314:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762315:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762317:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762318:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762319:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762320:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762322:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762323:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762324:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762324:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762325:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762326:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762327:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762331:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762332:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762334:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762335:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762338:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762339:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762340:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762341:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762342:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762344:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762345:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762346:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762347:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762348:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762349:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762351:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762352:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762353:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762354:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762356:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762356:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762358:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762358:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762359:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762360:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762361:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762362:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762362:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762364:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762365:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762366:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762367:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762372:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762373:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762375:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762376:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762377:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762378:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762379:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762380:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762381:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762382:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762382:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762384:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762384:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762385:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762386:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762387:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762388:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762390:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762391:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762392:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762392:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762394:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762395:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762396:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762397:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762398:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762399:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762400:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762401:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762402:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762404:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762405:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762407:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762410:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762412:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762413:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762414:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762415:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762416:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762417:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762417:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762418:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762419:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762420:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762421:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762422:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762423:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762425:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762426:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762427:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762428:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762429:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762430:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762431:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762431:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762432:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762433:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762434:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762435:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762436:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762437:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762438:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762439:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762440:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762441:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762442:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762445:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762446:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762447:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762448:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762449:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762450:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762451:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762452:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762453:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762454:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762455:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762457:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762458:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762460:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762460:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762461:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762462:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762463:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762464:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762464:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762465:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762466:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762467:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762468:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762469:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762470:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762472:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762472:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762473:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762474:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762475:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762476:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762477:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762480:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762480:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762482:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762483:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762484:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762485:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762486:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762487:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762489:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762489:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762491:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762491:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762493:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762494:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762494:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762495:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762496:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762497:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762498:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762499:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762500:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762501:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762502:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762503:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762504:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762505:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762505:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762506:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762507:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762508:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762509:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762509:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762510:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762515:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762516:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762517:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762518:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762519:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762521:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762522:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762523:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762524:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762525:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762526:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762527:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762527:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762528:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762529:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762530:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762531:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762532:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762533:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762534:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762535:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762536:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762537:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762538:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762539:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762540:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762541:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762541:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762542:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762543:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762544:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762545:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762546:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762549:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762550:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762552:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762553:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762555:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762555:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762557:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762558:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762558:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762559:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762560:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762561:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762562:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762563:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762564:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762565:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762566:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762567:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762568:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762569:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762570:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762571:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762572:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762572:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762573:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762574:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762575:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762576:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762577:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762578:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762579:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762580:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762581:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762584:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762586:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762587:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762588:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762589:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762590:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762590:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762591:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762592:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762593:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762594:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762595:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762596:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762597:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762599:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762600:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762601:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762601:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762602:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762603:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762604:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762605:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762605:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762607:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762607:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762608:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762609:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762610:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762611:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762613:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762613:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762615:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762615:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762618:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762619:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762620:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762621:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762622:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762623:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762624:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762625:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762626:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762627:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762628:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762630:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762631:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762632:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762633:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762634:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762635:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762635:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762636:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762637:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762638:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762639:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762640:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762641:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762642:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762642:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762644:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762645:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762646:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762646:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762647:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762648:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762649:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762652:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762652:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762654:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762655:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762656:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762657:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762658:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762659:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762661:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762662:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762663:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762664:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762665:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762666:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762667:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762667:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762668:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762669:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762670:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762671:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762672:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762673:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762674:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762675:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762676:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762677:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762678:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762678:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762679:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762680:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762681:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762681:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762683:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762686:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762687:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762688:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762689:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762690:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762692:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762693:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762694:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762695:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762696:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762697:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762698:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762698:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762699:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762700:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762701:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762723:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762725:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762727:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762729:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762731:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762732:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762734:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762734:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762736:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762737:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762738:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762739:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762740:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762742:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762743:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762744:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762745:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762750:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762751:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762754:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762755:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762757:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762757:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762759:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762760:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762761:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762761:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762762:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762763:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762764:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762765:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762766:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762767:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762768:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762770:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762770:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762772:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762772:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762773:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762774:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762775:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762775:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762776:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762777:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762778:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762779:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762780:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762781:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762782:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762783:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762787:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762789:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762790:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762791:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762792:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762793:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762794:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762794:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762795:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762796:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762797:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762798:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762799:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762800:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762802:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762803:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762804:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762805:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762806:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762807:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762808:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762808:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762809:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762810:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762811:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762812:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762813:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762814:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762815:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762816:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762817:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762818:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762819:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762822:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762823:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762824:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762825:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762826:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762827:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762828:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762829:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762830:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762831:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762832:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762834:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762835:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762836:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762837:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762838:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762839:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762840:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762840:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762841:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762842:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762843:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762844:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762845:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762846:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762847:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762848:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762849:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762850:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762851:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762852:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762853:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762853:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762856:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762857:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762858:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762859:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762860:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762861:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762862:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762863:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762866:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762866:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762868:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762868:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762869:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762870:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762871:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762872:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762872:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762873:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762874:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762875:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762876:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762877:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762878:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762880:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762880:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762881:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762882:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762883:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762884:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762885:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762885:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762886:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762887:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762890:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762891:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762892:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762893:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762894:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762897:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762897:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762899:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762899:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762900:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762901:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762902:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762903:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762903:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762905:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762906:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762907:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762908:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762909:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762910:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762911:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762912:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762913:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762913:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762914:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762915:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762916:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762917:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762917:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762918:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762919:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762920:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762921:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762925:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762926:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762928:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762929:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762930:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762931:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762932:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762933:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762934:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762934:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762935:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762936:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762937:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762938:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762939:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762940:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762941:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762942:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762943:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762944:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762945:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762946:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762947:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762948:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762948:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762949:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762950:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762951:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762952:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762953:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762954:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762955:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762956:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762959:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762961:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762961:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762962:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762963:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762964:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762965:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762966:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762967:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762968:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762969:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762970:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762971:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762972:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762974:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762974:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762976:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762976:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762977:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762978:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762979:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762979:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762980:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762981:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762982:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762983:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.762984:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.762985:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762986:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762987:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.762988:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.762989:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.762990:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.762993:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.762994:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.762995:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.762996:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.762997:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.762998:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.762999:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763000:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.763001:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.763002:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763003:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763005:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.763006:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.763007:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.763008:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.763009:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.763010:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.763010:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.763011:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.763012:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.763013:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763014:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763015:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.763016:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.763017:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763018:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763019:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.763020:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.763021:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.763021:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.763022:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.763023:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.763024:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.763028:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.763028:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.763030:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763031:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763032:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.763033:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.763034:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763035:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763037:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.763038:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.763039:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.763040:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.763041:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.763042:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.763043:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.763043:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.763044:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.763045:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763046:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763047:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.763048:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.763049:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763050:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763051:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.763052:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.763053:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.763054:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.763055:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.763056:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.763057:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.763057:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.763058:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.763059:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763062:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763063:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.763064:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.763065:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763066:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763069:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.763069:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.763071:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.763072:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.763073:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.763074:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.763074:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.763075:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.763076:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.763077:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763078:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763079:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.763080:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.763081:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763082:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763083:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.763084:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.763085:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.763085:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.763086:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.763087:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.763088:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.763089:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.763089:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.763091:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763091:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763092:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.763093:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.763097:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763098:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763100:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.763101:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.763103:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.763103:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.763104:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.763105:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.763106:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.763107:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.763108:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.763109:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763110:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763111:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.763112:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.763113:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763114:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763115:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.763116:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.763117:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.763117:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.763118:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.763119:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.763120:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.763121:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.763121:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.763122:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763123:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763124:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.763125:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.763126:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763127:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763129:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.763132:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.763133:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.763134:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.763135:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.763136:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.763137:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.763137:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.763138:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.763139:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763140:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763141:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.763142:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.763143:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763144:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763146:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.763147:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.763148:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.763149:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.763149:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.763150:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.763151:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.763152:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.763152:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.763154:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763155:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763155:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.763157:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.763157:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763158:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763160:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.763160:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.763162:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.763162:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.763165:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.763167:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.763167:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.763168:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.763169:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.763170:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763171:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763172:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.763173:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.763174:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763175:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763177:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.763178:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.763179:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.763180:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.763181:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.763182:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.763182:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.763183:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.763184:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.763185:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763186:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763187:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.763188:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.763189:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763190:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763191:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.763192:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.763193:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.763193:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.763194:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.763195:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.763196:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.763199:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.763200:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.763201:0:12084:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763202:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763203:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:1.0:1773941420.763204:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.763205:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763206:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:1.0:1773941420.763208:0:12084:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:1.0:1773941420.763209:0:12084:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) cmd=c00866a4 arg=7ffcd7379a30 00000080:00000001:1.0:1773941420.763210:0:12084:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:1.0:1773941420.763211:0:12084:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:1.0:1773941420.763212:0:12084:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b5ba0734400. 00000020:00000001:1.0:1773941420.763213:0:12084:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.763214:0:12084:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:1.0:1773941420.763214:0:12084:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:1.0:1773941420.763215:0:12084:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b5b885e5800: cmd=c00866a4 len=592 karg=ffff8b5ba0734400 uarg=0000000000000000 00020000:00000001:1.0:1773941420.763216:0:12084:0:(lov_obd.c:984:lov_iocontrol()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:1.0:1773941420.763217:0:12084:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:1.0:1773941420.763218:0:12084:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551597 : -19 : 0xffffffffffffffed) 00000080:00000010:1.0:1773941420.763219:0:12084:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff8b5ba0734400. 00000080:00000001:1.0:1773941420.763220:0:12084:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:1.0:1773941420.763221:0:12084:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:1.0:1773941420.763228:0:12084:0:(dir.c:2980:ll_dir_release()) Process entered 00000080:00000001:1.0:1773941420.763229:0:12084:0:(file.c:420:ll_file_release()) Process entered 00000020:00000001:1.0:1773941420.763231:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697db0 00000080:00200008:1.0:1773941420.763232:0:12084:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b5b83538908), flags 100000 00000080:00400000:1.0:1773941420.763234:0:12084:0:(statahead.c:1714:ll_deauthorize_statahead()) deauthorize statahead for [0x200000007:0x1:0x0] 00000080:00000010:1.0:1773941420.763237:0:12084:0:(file.c:80:ll_file_data_put()) slab-freed '(lfd)': 288 at ffff8b5b817a7e00. 00000080:00000001:1.0:1773941420.763239:0:12084:0:(file.c:439:ll_file_release()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1773941420.763241:0:12084:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41697db0 00000080:00000008:1.0:1773941420.763242:0:12084:0:(file.c:464:ll_file_release()) COMPLETED file /:[0x200000007:0x1:0x0](ffff8b5b83538908), flags 100000, rc = 0 00000080:00000001:1.0:1773941420.763243:0:12084:0:(file.c:466:ll_file_release()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:1.0:1773941420.763247:0:12084:0:(dir.c:2981:ll_dir_release()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941420.772430:0:12100:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41637d08 00000080:00200008:2.0:1773941420.772436:0:12100:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b5b83538908), request_mask 2047, flags 0, foreign 0 00000080:00000001:2.0:1773941420.772444:0:12100:0:(file.c:6184:ll_inode_revalidate()) Process entered 00000020:00000001:2.0:1773941420.772446:0:12100:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41637c80 00000080:00200000:2.0:1773941420.772448:0:12100:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908),name=/ 00000080:00000010:2.0:1773941420.772454:0:12100:0:(llite_lib.c:3907:ll_prep_md_op_data()) kmalloced '(op_data)': 384 at ffff8b5b89317a00. 00800000:00000002:2.0:1773941420.772461:0:12100:0:(lmv_obd.c:3847:lmv_stripe_object_get()) get ffff8b5b850b0640 2 00000080:00000001:2.0:1773941420.772469:0:12100:0:(namei.c:943:ll_intent_lock()) Process entered 00800000:00000001:2.0:1773941420.772473:0:12100:0:(lmv_intent.c:556:lmv_intent_lock()) Process entered 00000020:00000001:2.0:1773941420.772474:0:12100:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41637ba0 00800000:00000002:2.0:1773941420.772477:0:12100:0:(lmv_intent.c:563:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:2.0:1773941420.772480:0:12100:0:(lmv_intent.c:437:lmv_intent_lookup()) Process entered 00800000:00000002:2.0:1773941420.772483:0:12100:0:(lmv_intent.c:490:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:2.0:1773941420.772486:0:12100:0:(mdc_locks.c:1500:mdc_intent_lock()) Process entered 00000020:00000001:2.0:1773941420.772488:0:12100:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41637a70 00000002:00010000:2.0:1773941420.772489:0:12100:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:2.0:1773941420.772492:0:12100:0:(mdc_locks.c:1404:mdc_revalidate_lock()) Process entered 00000002:00000001:2.0:1773941420.772494:0:12100:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1773941420.772496:0:12100:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1773941420.772502:0:12100:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8b5b890ec400 count: 2 00010000:00000001:2.0:1773941420.772504:0:12100:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941420.772508:0:12100:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000020:00000010:2.0:1773941420.772512:0:12100:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff8b5b89390500. 00010000:00000001:2.0:1773941420.772514:0:12100:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:2.0:1773941420.772518:0:12100:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 02000000:00000001:2.0:1773941420.772526:0:12100:0:(sec.c:1015:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:2.0:1773941420.772529:0:12100:0:(sec.c:1031:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941420.772532:0:12100:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000002:00000001:2.0:1773941420.772537:0:12100:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:2.0:1773941420.772538:0:12100:0:(mdc_locks.c:1456:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:2.0:1773941420.772540:0:12100:0:(mdc_locks.c:1523:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:2.0:1773941420.772541:0:12100:0:(lmv_intent.c:511:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:2.0:1773941420.772543:0:12100:0:(lmv_intent.c:596:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:2.0:1773941420.772544:0:12100:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00000001:2.0:1773941420.772546:0:12100:0:(namei.c:989:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:2.0:1773941420.772548:0:12100:0:(lmv_obd.c:3892:lmv_stripe_object_put()) put ffff8b5b850b0640 2 00000080:00000010:2.0:1773941420.772551:0:12100:0:(llite_lib.c:4048:ll_finish_md_op_data()) kfreed 'op_data': 384 at ffff8b5b89317a00. 00000080:00000001:2.0:1773941420.772558:0:12100:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:2.0:1773941420.772559:0:12100:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:2.0:1773941420.772561:0:12100:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) 00000080:00010000:2.0:1773941420.772563:0:12100:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) for lock 0xa33a5be14eae1c36 00800000:00000001:2.0:1773941420.772566:0:12100:0:(lmv_obd.c:3932:lmv_set_lock_data()) Process entered 00000002:00000001:2.0:1773941420.772567:0:12100:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:2.0:1773941420.772569:0:12100:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941420.772571:0:12100:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941420.772572:0:12100:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=3 00000020:00000001:2.0:1773941420.772574:0:12100:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:2.0:1773941420.772576:0:12100:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:2.0:1773941420.772578:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941420.772579:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:2.0:1773941420.772580:0:12100:0:(mdc_locks.c:122:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:2.0:1773941420.772582:0:12100:0:(lmv_obd.c:3937:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:2.0:1773941420.772583:0:12100:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xa33a5be14eae1c36 from it ffffa10c41637cb8 00010000:00000001:2.0:1773941420.772585:0:12100:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941420.772586:0:12100:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941420.772587:0:12100:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=3 00000020:00000001:2.0:1773941420.772588:0:12100:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:2.0:1773941420.772589:0:12100:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:2.0:1773941420.772591:0:12100:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:2.0:1773941420.772593:0:12100:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00000001:2.0:1773941420.772597:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941420.772598:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941420.772599:0:12100:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:2.0:1773941420.772600:0:12100:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:2.0:1773941420.772602:0:12100:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00000001:2.0:1773941420.772607:0:12100:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1773941420.772612:0:12100:0:(ldlm_request.c:2131:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1773941420.772614:0:12100:0:(ldlm_request.c:1927:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1773941420.772616:0:12100:0:(ldlm_request.c:2099:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.772619:0:12100:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1773941420.772620:0:12100:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.772622:0:12100:0:(ldlm_request.c:2141:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941420.772624:0:12100:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 448 at ffff8b5b89317a00. 00010000:00000001:2.0:1773941420.772627:0:12100:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.772629:0:12100:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941420.772631:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941420.772632:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941420.772635:0:12100:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941420.772637:0:12100:0:(client.c:2844:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1773941420.772641:0:12100:0:(file.c:6244:ll_merge_md_attr()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941420.772647:0:12100:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 1856 at ffff8b5b90f22800. 00000020:00000001:2.0:1773941420.772650:0:12100:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41637d08 00000080:00000008:2.0:1773941420.772653:0:12100:0:(file.c:6469:ll_getattr_dentry()) COMPLETED file /:[0x200000007:0x1:0x0](ffff8b5b83538908), request_mask 2047, flags 0, foreign 0 00000020:00000001:2.0:1773941420.772799:0:12100:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41637d08 00000080:00200008:2.0:1773941420.772801:0:12100:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b5b83538908), request_mask 2047, flags 0, foreign 0 00000080:00000001:2.0:1773941420.772805:0:12100:0:(file.c:6184:ll_inode_revalidate()) Process entered 00000020:00000001:2.0:1773941420.772807:0:12100:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41637c80 00000080:00200000:2.0:1773941420.772808:0:12100:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908),name=/ 00000080:00000010:2.0:1773941420.772811:0:12100:0:(llite_lib.c:3907:ll_prep_md_op_data()) kmalloced '(op_data)': 384 at ffff8b5b89317600. 00800000:00000002:2.0:1773941420.772813:0:12100:0:(lmv_obd.c:3847:lmv_stripe_object_get()) get ffff8b5b850b0640 2 00000080:00000001:2.0:1773941420.772815:0:12100:0:(namei.c:943:ll_intent_lock()) Process entered 00800000:00000001:2.0:1773941420.772817:0:12100:0:(lmv_intent.c:556:lmv_intent_lock()) Process entered 00000020:00000001:2.0:1773941420.772819:0:12100:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41637ba0 00800000:00000002:2.0:1773941420.772820:0:12100:0:(lmv_intent.c:563:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:2.0:1773941420.772822:0:12100:0:(lmv_intent.c:437:lmv_intent_lookup()) Process entered 00800000:00000002:2.0:1773941420.772824:0:12100:0:(lmv_intent.c:490:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:2.0:1773941420.772826:0:12100:0:(mdc_locks.c:1500:mdc_intent_lock()) Process entered 00000020:00000001:2.0:1773941420.772828:0:12100:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41637a70 00000002:00010000:2.0:1773941420.772829:0:12100:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:2.0:1773941420.772831:0:12100:0:(mdc_locks.c:1404:mdc_revalidate_lock()) Process entered 00000002:00000001:2.0:1773941420.772832:0:12100:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1773941420.772838:0:12100:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1773941420.772841:0:12100:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8b5b890ec400 count: 2 00010000:00000001:2.0:1773941420.772843:0:12100:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941420.772845:0:12100:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941420.772847:0:12100:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:2.0:1773941420.772850:0:12100:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 02000000:00000001:2.0:1773941420.772855:0:12100:0:(sec.c:1015:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:2.0:1773941420.772857:0:12100:0:(sec.c:1031:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941420.772859:0:12100:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000002:00000001:2.0:1773941420.772863:0:12100:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:2.0:1773941420.772865:0:12100:0:(mdc_locks.c:1456:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:2.0:1773941420.772866:0:12100:0:(mdc_locks.c:1523:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:2.0:1773941420.772867:0:12100:0:(lmv_intent.c:511:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:2.0:1773941420.772869:0:12100:0:(lmv_intent.c:596:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:2.0:1773941420.772870:0:12100:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00000001:2.0:1773941420.772872:0:12100:0:(namei.c:989:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:2.0:1773941420.772874:0:12100:0:(lmv_obd.c:3892:lmv_stripe_object_put()) put ffff8b5b850b0640 2 00000080:00000010:2.0:1773941420.772875:0:12100:0:(llite_lib.c:4048:ll_finish_md_op_data()) kfreed 'op_data': 384 at ffff8b5b89317600. 00000080:00000001:2.0:1773941420.772878:0:12100:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:2.0:1773941420.772880:0:12100:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:2.0:1773941420.772882:0:12100:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) 00000080:00010000:2.0:1773941420.772884:0:12100:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b5b83538908) for lock 0xa33a5be14eae1c36 00800000:00000001:2.0:1773941420.772886:0:12100:0:(lmv_obd.c:3932:lmv_set_lock_data()) Process entered 00000002:00000001:2.0:1773941420.772887:0:12100:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:2.0:1773941420.772889:0:12100:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941420.772890:0:12100:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941420.772892:0:12100:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=3 00000020:00000001:2.0:1773941420.772893:0:12100:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:2.0:1773941420.772895:0:12100:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:2.0:1773941420.772897:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941420.772902:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:2.0:1773941420.772903:0:12100:0:(mdc_locks.c:122:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:2.0:1773941420.772905:0:12100:0:(lmv_obd.c:3937:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:2.0:1773941420.772907:0:12100:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xa33a5be14eae1c36 from it ffffa10c41637cb8 00010000:00000001:2.0:1773941420.772908:0:12100:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941420.772909:0:12100:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941420.772910:0:12100:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ea00 refcount=3 00000020:00000001:2.0:1773941420.772912:0:12100:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:2.0:1773941420.772914:0:12100:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859116544 : -128249850435072 : ffff8b5b8138ea00) 00010000:00000001:2.0:1773941420.772915:0:12100:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:2.0:1773941420.772917:0:12100:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00000001:2.0:1773941420.772921:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941420.772922:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941420.772923:0:12100:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:2.0:1773941420.772925:0:12100:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:2.0:1773941420.772927:0:12100:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00000001:2.0:1773941420.772931:0:12100:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1773941420.772932:0:12100:0:(ldlm_request.c:2131:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1773941420.772933:0:12100:0:(ldlm_request.c:1927:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1773941420.772935:0:12100:0:(ldlm_request.c:2099:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.772937:0:12100:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1773941420.772938:0:12100:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.772939:0:12100:0:(ldlm_request.c:2141:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.772941:0:12100:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.772942:0:12100:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941420.772943:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941420.772944:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941420.772945:0:12100:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941420.772946:0:12100:0:(client.c:2844:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1773941420.772948:0:12100:0:(file.c:6244:ll_merge_md_attr()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941420.772950:0:12100:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffa10c41637d08 00000080:00000008:2.0:1773941420.772951:0:12100:0:(file.c:6469:ll_getattr_dentry()) COMPLETED file /:[0x200000007:0x1:0x0](ffff8b5b83538908), request_mask 2047, flags 0, foreign 0 00000080:00000001:2.0:1773941420.773286:0:12100:0:(llite_lib.c:4092:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1773941420.782781:0:12100:0:(llite_lib.c:1002:ll_kill_super()) Process entered 00000080:00000001:2.0:1773941420.782784:0:12100:0:(llite_lib.c:1023:ll_kill_super()) Process leaving 00000080:00000001:2.0:1773941420.782791:0:12100:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1773941420.782793:0:12100:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1773941420.782801:0:12100:0:(llite_lib.c:3261:ll_delete_inode()) Process entered 00000080:00000001:2.0:1773941420.782803:0:12100:0:(llite_lib.c:1948:ll_clear_inode()) Process entered 00000080:00200000:2.0:1773941420.782805:0:12100:0:(llite_lib.c:1951:ll_clear_inode()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b5b83538908) 00800000:00000001:2.0:1773941420.782810:0:12100:0:(lmv_obd.c:1610:lmv_null_inode()) Process entered 00800000:00000002:2.0:1773941420.782811:0:12100:0:(lmv_obd.c:1612:lmv_null_inode()) CBDATA for [0x200000007:0x1:0x0] 00000002:00000001:2.0:1773941420.782814:0:12100:0:(mdc_locks.c:166:mdc_null_inode()) Process entered 00010000:00000040:2.0:1773941420.782819:0:12100:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8b5b890ec400 count: 2 00010000:00000001:2.0:1773941420.782822:0:12100:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1773941420.782824:0:12100:0:(mdc_locks.c:180:mdc_null_inode()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:2.0:1773941420.782826:0:12100:0:(lmv_obd.c:1622:lmv_null_inode()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1773941420.782827:0:12100:0:(xattr_cache.c:319:ll_xattr_cache_destroy()) Process entered 00000080:00000001:2.0:1773941420.782829:0:12100:0:(xattr_cache.c:300:ll_xattr_cache_destroy_locked()) Process entered 00000080:00000001:2.0:1773941420.782831:0:12100:0:(xattr_cache.c:303:ll_xattr_cache_destroy_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1773941420.782832:0:12100:0:(xattr_cache.c:325:ll_xattr_cache_destroy()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:2.0:1773941420.782835:0:12100:0:(lmv_obd.c:3892:lmv_stripe_object_put()) put ffff8b5b850b0640 1 00800000:00000010:2.0:1773941420.782837:0:12100:0:(lmv_obd.c:3877:lmv_stripe_object_free()) kfreed 'lsm_obj': 56 at ffff8b5b850b0640. 00000080:00000001:2.0:1773941420.782840:0:12100:0:(llite_lib.c:1999:ll_clear_inode()) Process leaving 00000080:00000001:2.0:1773941420.782841:0:12100:0:(llite_lib.c:3280:ll_delete_inode()) Process leaving 00000080:00000001:2.0:1773941420.782846:0:12100:0:(llite_lib.c:1565:ll_put_super()) Process entered 00000080:00200000:2.0:1773941420.782848:0:12100:0:(llite_lib.c:1572:ll_put_super()) VFS Op: cfg_instance lustre-client-ffff8b5b885e5800 (sb ffff8b5b885e5800) 00000020:00000001:2.0:1773941420.782851:0:12100:0:(obd_mount.c:116:lustre_end_log()) Process entered 00000020:00000010:2.0:1773941420.782854:0:12100:0:(obd_mount.c:126:lustre_end_log()) kmalloced '(lcfg)': 176 at ffff8b5ba0b54a80. 00000020:00000001:2.0:1773941420.782856:0:12100:0:(obd_class.h:681:obd_process_config()) Process entered 00000020:00000010:2.0:1773941420.782858:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b89316600. 00000020:00000010:2.0:1773941420.782862:0:12100:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b89316400. 00000040:00000010:2.0:1773941420.782864:0:12100:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b89317000. 00000020:00000010:2.0:1773941420.782868:0:12100:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90f26000. 00000020:00000010:2.0:1773941420.782871:0:12100:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b89317400. 00040000:00000010:2.0:1773941420.782875:0:12100:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98bb0900. 00040000:00000010:2.0:1773941420.782878:0:12100:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b89316c00. 10000000:00000001:2.0:1773941420.782881:0:12100:0:(mgc_request.c:1932:mgc_process_config()) Process entered 10000000:00000001:2.0:1773941420.782887:0:12100:0:(mgc_request_server.c:668:mgc_process_config_server()) Process entered 10000000:00000001:2.0:1773941420.782889:0:12100:0:(mgc_request.c:493:config_log_end()) Process entered 10000000:00000001:2.0:1773941420.782891:0:12100:0:(mgc_request.c:159:config_log_find()) Process entered 10000000:00000001:2.0:1773941420.782892:0:12100:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941420.782893:0:12100:0:(mgc_request.c:108:config_log_get()) log lustre-client (ffff8b5b8809da00) refs 3 10000000:00000001:2.0:1773941420.782895:0:12100:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941420.782896:0:12100:0:(mgc_request.c:177:config_log_find()) Process leaving (rc=18446615823973472768 : -128249736078848 : ffff8b5b8809da00) 10000000:00000001:2.0:1773941420.782898:0:12100:0:(mgc_request.c:461:config_mark_cld_stop_nolock()) Process entered 10000000:00000040:2.0:1773941420.782899:0:12100:0:(mgc_request.c:467:config_mark_cld_stop_nolock()) lockh 0xa33a5be14eae1be9 00010000:00000001:2.0:1773941420.782901:0:12100:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941420.782903:0:12100:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941420.782904:0:12100:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484a00 refcount=2 00000020:00000001:2.0:1773941420.782906:0:12100:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00010000:00000001:2.0:1773941420.782908:0:12100:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00010000:00000001:2.0:1773941420.782910:0:12100:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941420.782912:0:12100:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941420.782916:0:12100:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1be9 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xb574482a8fd85f7b expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941420.782921:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941420.782923:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941420.782924:0:12100:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941420.782924:0:12100:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941420.782925:0:12100:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484a00 refcount=3 00000020:00000001:2.0:1773941420.782926:0:12100:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00010000:00000001:2.0:1773941420.782927:0:12100:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00010000:00000001:2.0:1773941420.782929:0:12100:0:(ldlm_lock.c:950:ldlm_lock_decref_and_cancel()) Process entered 00010000:00010000:2.0:1773941420.782930:0:12100:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1be9 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0xb574482a8fd85f7b expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941420.782934:0:12100:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:2.0:1773941420.782935:0:12100:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1be9 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000400000000 nid: local remote: 0xb574482a8fd85f7b expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941420.782943:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941420.782944:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941420.782945:0:12100:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1be9 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000400000000 nid: local remote: 0xb574482a8fd85f7b expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941420.782948:0:12100:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941420.782949:0:12100:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.782951:0:12100:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000010:2.0:1773941420.782954:0:12100:0:(ldlm_lockd.c:2249:ldlm_bl_to_thread()) kmalloced '(blwi)': 192 at ffff8b5ba0b54cc0. 00010000:00000001:2.0:1773941420.782956:0:12100:0:(ldlm_lockd.c:2159:__ldlm_bl_to_thread()) Process entered 00010000:00010000:2.0:1773941420.782964:0:12100:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00000001:2.0:1773941420.782966:0:12100:0:(ldlm_lockd.c:2198:__ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.782967:0:12100:0:(ldlm_lockd.c:2254:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.782968:0:12100:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941420.782969:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941420.782970:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 10000000:00000001:2.0:1773941420.782971:0:12100:0:(mgc_request.c:461:config_mark_cld_stop_nolock()) Process entered 10000000:00000040:2.0:1773941420.782972:0:12100:0:(mgc_request.c:467:config_mark_cld_stop_nolock()) lockh 0xa33a5be14eae1c0c 00010000:00000001:2.0:1773941420.782973:0:12100:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941420.782974:0:12100:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941420.782976:0:12100:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484600 refcount=2 00000020:00000001:0.0:1773941420.782977:0:12029:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941420.782978:0:12100:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00000020:00000001:0.0:1773941420.782981:0:12029:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.782982:0:12100:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:2.0:1773941420.782987:0:12100:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00010000:0.0:1773941420.782987:0:12029:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00000001:2.0:1773941420.782988:0:12100:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941420.782991:0:12029:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 00010000:00010000:2.0:1773941420.782992:0:12100:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c0c lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb574482a8fd85f82 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941420.782994:0:12029:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00000001:2.0:1773941420.782999:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1773941420.782999:0:12029:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1be9 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000400000000 nid: local remote: 0xb574482a8fd85f7b expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941420.783002:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941420.783004:0:12100:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941420.783006:0:12100:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941420.783008:0:12100:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484600 refcount=3 00010000:00010000:0.0:1773941420.783011:0:12029:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81484a00 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:0.0:1773941420.783014:0:12029:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000020:00000001:2.0:1773941420.783015:0:12100:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:2.0:1773941420.783018:0:12100:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 10000000:00010000:0.0:1773941420.783018:0:12029:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1be9 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000400000000 nid: local remote: 0xb574482a8fd85f7b expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941420.783023:0:12100:0:(ldlm_lock.c:950:ldlm_lock_decref_and_cancel()) Process entered 00010000:00010000:2.0:1773941420.783026:0:12100:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c0c lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xb574482a8fd85f82 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941420.783027:0:12029:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1773941420.783030:0:12029:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941420.783032:0:12029:0:(lustre_handles.c:113:class_handle2object()) Process entered 00010000:00000001:2.0:1773941420.783034:0:12100:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000020:00000040:0.0:1773941420.783035:0:12029:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484a00 refcount=3 00010000:00010000:2.0:1773941420.783037:0:12100:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c0c lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xb574482a8fd85f82 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1773941420.783038:0:12029:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00010000:00000001:0.0:1773941420.783042:0:12029:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00010000:00000001:0.0:1773941420.783046:0:12029:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00000001:2.0:1773941420.783047:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941420.783048:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941420.783049:0:12100:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c0c lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xb574482a8fd85f82 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941420.783052:0:12100:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941420.783055:0:12100:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941420.783056:0:12029:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1be9 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1008400000000 nid: local remote: 0xb574482a8fd85f7b expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941420.783057:0:12100:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000010:2.0:1773941420.783061:0:12100:0:(ldlm_lockd.c:2249:ldlm_bl_to_thread()) kmalloced '(blwi)': 192 at ffff8b5ba0b543c0. 00010000:00000001:2.0:1773941420.783065:0:12100:0:(ldlm_lockd.c:2159:__ldlm_bl_to_thread()) Process entered 10000000:00000001:0.0:1773941420.783065:0:12029:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1773941420.783068:0:12029:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1be9 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1009400000000 nid: local remote: 0xb574482a8fd85f7b expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1773941420.783072:0:12029:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1773941420.783074:0:12100:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 10000000:01000000:0.0:1773941420.783075:0:12029:0:(mgc_request.c:944:mgc_blocking_ast()) log lustre-client: stopping, won't requeue 10000000:00000001:0.0:1773941420.783077:0:12029:0:(mgc_request.c:117:config_log_put()) Process entered 00010000:00000001:2.0:1773941420.783080:0:12100:0:(ldlm_lockd.c:2198:__ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:0.0:1773941420.783080:0:12029:0:(mgc_request.c:123:config_log_put()) log lustre-client(ffff8b5b8809da00) refs 3 10000000:00000001:0.0:1773941420.783086:0:12029:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000001:2.0:1773941420.783087:0:12100:0:(ldlm_lockd.c:2254:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941420.783088:0:12029:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.783092:0:12100:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941420.783093:0:12029:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000020:00000001:3.0:1773941420.783095:0:12028:0:(genops.c:1914:obd_stale_export_get()) Process entered 00010000:00000001:2.0:1773941420.783096:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941420.783097:0:12029:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000020:00000001:3.0:1773941420.783099:0:12028:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.783099:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941420.783100:0:12029:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00010000:3.0:1773941420.783102:0:12028:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 10000000:00000001:2.0:1773941420.783102:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 00010000:00000001:0.0:1773941420.783103:0:12029:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 10000000:00000040:2.0:1773941420.783105:0:12100:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff8b5b8809c800) refs 2 00010000:00000001:0.0:1773941420.783107:0:12029:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941420.783111:0:12100:0:(mgc_request.c:147:config_log_put()) Process leaving 00000020:00000040:0.0:1773941420.783112:0:12029:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81484a00 with handle 0xa33a5be14eae1be9 from hash 10000000:00000001:2.0:1773941420.783114:0:12100:0:(mgc_request.c:461:config_mark_cld_stop_nolock()) Process entered 00010000:00000001:0.0:1773941420.783115:0:12029:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 10000000:00000040:2.0:1773941420.783119:0:12100:0:(mgc_request.c:467:config_mark_cld_stop_nolock()) lockh 0xa33a5be14eae1c13 00010000:00000001:0.0:1773941420.783119:0:12029:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941420.783121:0:12029:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941420.783124:0:12100:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:0.0:1773941420.783124:0:12029:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1773941420.783125:0:12029:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000020:00000001:2.0:1773941420.783128:0:12100:0:(lustre_handles.c:113:class_handle2object()) Process entered 00010000:00000001:0.0:1773941420.783129:0:12029:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00000020:00000040:2.0:1773941420.783133:0:12100:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138fa00 refcount=2 00010000:00000001:0.0:1773941420.783133:0:12029:0:(ldlm_request.c:2242:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1773941420.783136:0:12029:0:(ldlm_request.c:1396:ldlm_cli_cancel_req()) Process entered 00000020:00000001:2.0:1773941420.783137:0:12100:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859120640 : -128249850430976 : ffff8b5b8138fa00) 00000100:00000010:0.0:1773941420.783142:0:12029:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b850a8a80. 00000020:00000040:0.0:1773941420.783145:0:12029:0:(genops.c:1229:class_import_get()) import ffff8b5b88954000 refcount=4 obd=MGC192.168.204.143@tcp 00010000:00000001:2.0:1773941420.783150:0:12100:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859120640 : -128249850430976 : ffff8b5b8138fa00) 00000100:00000001:0.0:1773941420.783151:0:12029:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941420.783155:0:12029:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 00010000:00000001:2.0:1773941420.783157:0:12100:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 02000000:00000001:0.0:1773941420.783159:0:12029:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.783161:0:12100:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941420.783165:0:12029:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b88908a00. 00010000:00010000:2.0:1773941420.783166:0:12100:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1c13 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb574482a8fd85f89 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941420.783173:0:12029:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.783175:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941420.783176:0:12029:0:(ldlm_request.c:1348:ldlm_cancel_pack()) Process entered 00010000:00000001:2.0:1773941420.783178:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941420.783179:0:12100:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941420.783181:0:12100:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941420.783184:0:12100:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138fa00 refcount=3 00000020:00000001:2.0:1773941420.783187:0:12100:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859120640 : -128249850430976 : ffff8b5b8138fa00) 00010000:00010000:0.0:1773941420.783188:0:12029:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1be9 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85f7b expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941420.783191:0:12100:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859120640 : -128249850430976 : ffff8b5b8138fa00) 00010000:00010000:0.0:1773941420.783195:0:12029:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:2.0:1773941420.783197:0:12100:0:(ldlm_lock.c:950:ldlm_lock_decref_and_cancel()) Process entered 00010000:00000001:0.0:1773941420.783197:0:12029:0:(ldlm_request.c:1380:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1773941420.783202:0:12105:0:(ldlm_lockd.c:2959:ldlm_bl_thread_main()) Process entered 00010000:00010000:2.0:1773941420.783202:0:12100:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1c13 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xb574482a8fd85f89 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941420.783203:0:12029:0:(jobid.c:936:lustre_get_jobid()) Process entered 00010000:00000010:3.0:1773941420.783207:0:12105:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at ffff8b5b85068b80. 00000100:00000001:0.0:1773941420.783208:0:12029:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.783213:0:12105:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b91df5a00. 00010000:00000001:2.0:1773941420.783215:0:12100:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000020:00000010:3.0:1773941420.783217:0:12105:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b91df5800. 00010000:00010000:2.0:1773941420.783217:0:12100:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1c13 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xb574482a8fd85f89 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941420.783221:0:12029:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d2ea9673] to pc [ptlrpcd_00_00+0] req@ffff8b5b850a8a80 x1860112241143424/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1773941420.783223:0:12105:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b98ac6800. 00010000:00000001:2.0:1773941420.783228:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000020:00000010:3.0:1773941420.783229:0:12105:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b8849a840. 00010000:00000001:2.0:1773941420.783230:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941420.783234:0:12100:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1c13 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xb574482a8fd85f89 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000020:00000010:3.0:1773941420.783235:0:12105:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b91df5200. 80000000:00000010:3.0:1773941420.783241:0:12105:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc93c0. 00010000:00000001:2.0:1773941420.783241:0:12100:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941420.783243:0:12100:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 40000000:00000010:3.0:1773941420.783245:0:12105:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b850682c0. 00010000:00000001:2.0:1773941420.783245:0:12100:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000010:2.0:1773941420.783249:0:12100:0:(ldlm_lockd.c:2249:ldlm_bl_to_thread()) kmalloced '(blwi)': 192 at ffff8b5ba0b54480. 00010000:00000001:0.0:1773941420.783249:0:12029:0:(ldlm_request.c:1464:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 00100000:00000010:3.0:1773941420.783250:0:12105:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b98ac5000. 00010000:00000001:2.0:1773941420.783253:0:12100:0:(ldlm_lockd.c:2159:__ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1773941420.783255:0:12029:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00040000:00000010:3.0:1773941420.783257:0:12105:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b91e1e300. 00010000:00010000:2.0:1773941420.783257:0:12100:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00000001:0.0:1773941420.783258:0:12029:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941420.783261:0:12029:0:(ldlm_request.c:2290:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.783262:0:12100:0:(ldlm_lockd.c:2198:__ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:3.0:1773941420.783263:0:12105:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b91df5e00. 00010000:00000001:0.0:1773941420.783264:0:12029:0:(ldlm_request.c:1689:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941420.783278:0:12029:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941420.783281:0:12029:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1be9 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85f7b expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00002000:00000010:3.0:1773941420.783285:0:12105:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b90261800. 00010000:00000001:2.0:1773941420.783286:0:12100:0:(ldlm_lockd.c:2254:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.783291:0:12100:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941420.783292:0:12029:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000020:00000010:3.0:1773941420.783293:0:12105:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068e00. 00010000:00000001:2.0:1773941420.783293:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1773941420.783295:0:12029:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1be9 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85f7b expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941420.783296:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 10000000:00000001:2.0:1773941420.783299:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 00000020:00000001:3.0:1773941420.783303:0:12105:0:(genops.c:1914:obd_stale_export_get()) Process entered 10000000:00000040:2.0:1773941420.783303:0:12100:0:(mgc_request.c:123:config_log_put()) log params(ffff8b5b8809ca00) refs 2 00000020:00000001:3.0:1773941420.783305:0:12105:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941420.783306:0:12100:0:(mgc_request.c:147:config_log_put()) Process leaving 00000020:00000010:0.0:1773941420.783306:0:12029:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff8b5b8848f400. 00010000:00010000:3.0:1773941420.783307:0:12105:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 10000000:00000001:2.0:1773941420.783310:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941420.783311:0:12100:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941420.783313:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 00010000:00000001:3.0:1773941420.783315:0:12028:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 10000000:00000040:2.0:1773941420.783315:0:12100:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b839c6e00) refs 2 00010000:00000001:0.0:1773941420.783315:0:12029:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941420.783319:0:12028:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 10000000:00000001:2.0:1773941420.783319:0:12100:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000001:0.0:1773941420.783320:0:12029:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 10000000:00000001:2.0:1773941420.783323:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 00010000:00000010:0.0:1773941420.783324:0:12029:0:(ldlm_lockd.c:2913:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff8b5ba0b54cc0. 00010000:00010000:3.0:1773941420.783325:0:12028:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c0c lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xb574482a8fd85f82 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:2.0:1773941420.783325:0:12100:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941420.783328:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 00010000:00000001:0.0:1773941420.783328:0:12029:0:(ldlm_lockd.c:2917:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:2.0:1773941420.783330:0:12100:0:(mgc_request.c:123:config_log_put()) log lustre-client(ffff8b5b8809da00) refs 2 00000020:00000001:0.0:1773941420.783333:0:12029:0:(genops.c:1914:obd_stale_export_get()) Process entered 10000000:00000001:2.0:1773941420.783334:0:12100:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:2.0:1773941420.783337:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 00000020:00000001:0.0:1773941420.783337:0:12029:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:2.0:1773941420.783340:0:12100:0:(mgc_request.c:123:config_log_put()) log lustre-client(ffff8b5b8809da00) refs 1 00010000:00010000:0.0:1773941420.783342:0:12029:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:2.0:1773941420.783344:0:12100:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-client 00000020:00000001:0.0:1773941420.783345:0:12029:0:(genops.c:1914:obd_stale_export_get()) Process entered 10000000:00000001:2.0:1773941420.783347:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 00010000:00010000:3.0:1773941420.783349:0:12028:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81484600 already unused, calling callback (ffffffffc1482c20) 00000020:00000001:0.0:1773941420.783349:0:12029:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941420.783350:0:12100:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941420.783353:0:12028:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00000001:2.0:1773941420.783353:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 00010000:00010000:0.0:1773941420.783354:0:12029:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00010000:3.0:1773941420.783356:0:12028:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c0c lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xb574482a8fd85f82 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:2.0:1773941420.783356:0:12100:0:(mgc_request.c:120:config_log_put()) Process leaving 00000020:00000001:0.0:1773941420.783357:0:12029:0:(genops.c:1914:obd_stale_export_get()) Process entered 10000000:00000001:2.0:1773941420.783358:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941420.783360:0:12100:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941420.783361:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 00010000:00000001:3.0:1773941420.783362:0:12028:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 10000000:00000001:2.0:1773941420.783362:0:12100:0:(mgc_request.c:120:config_log_put()) Process leaving 00000020:00000001:0.0:1773941420.783365:0:12029:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.783366:0:12028:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 10000000:00000001:2.0:1773941420.783366:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 00000020:00000001:3.0:1773941420.783369:0:12028:0:(lustre_handles.c:113:class_handle2object()) Process entered 10000000:00000001:2.0:1773941420.783369:0:12100:0:(mgc_request.c:120:config_log_put()) Process leaving 00010000:00010000:0.0:1773941420.783370:0:12029:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000040:3.0:1773941420.783372:0:12028:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484600 refcount=3 00000020:00000040:2.0:1773941420.783373:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b885e2000 : new refcount 5 00000020:00000001:3.0:1773941420.783375:0:12028:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 10000000:00000010:2.0:1773941420.783375:0:12100:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 286 at ffff8b5b8809da00. 00010000:00000001:3.0:1773941420.783377:0:12028:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00000100:00000001:0.0:1773941420.783378:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000001:2.0:1773941420.783379:0:12100:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000001:3.0:1773941420.783380:0:12028:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 10000000:01000000:2.0:1773941420.783382:0:12100:0:(mgc_request.c:547:config_log_end()) end config log lustre-client (0) 00010000:00010000:3.0:1773941420.783383:0:12028:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c0c lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1008400000000 nid: local remote: 0xb574482a8fd85f82 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:2.0:1773941420.783385:0:12100:0:(mgc_request.c:548:config_log_end()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1773941420.783385:0:2352:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b902a4dc8. 10000000:00000001:2.0:1773941420.783390:0:12100:0:(mgc_request.c:2026:mgc_process_config()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941420.783392:0:12028:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000080:00000010:0.0:1773941420.783392:0:2352:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b8502c248. 00000020:00000010:2.0:1773941420.783394:0:12100:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b89316400. 10000000:00010000:3.0:1773941420.783395:0:12028:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c0c lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1009400000000 nid: local remote: 0xb574482a8fd85f82 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00000010:0.0:1773941420.783399:0:2352:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b902a21e0. 10000000:01000000:3.0:1773941420.783400:0:12028:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00000040:00000010:2.0:1773941420.783400:0:12100:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b89317000. 10000000:01000000:3.0:1773941420.783404:0:12028:0:(mgc_request.c:944:mgc_blocking_ast()) log lustre-cliir: stopping, won't requeue 00000100:00000001:0.0:1773941420.783404:0:2352:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000010:2.0:1773941420.783405:0:12100:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90f26000. 00000100:00000001:0.0:1773941420.783407:0:2352:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 10000000:00000001:3.0:1773941420.783408:0:12028:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941420.783412:0:12028:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff8b5b8809c800) refs 1 00000100:00000040:0.0:1773941420.783414:0:2352:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b850a8a80 x1860112241143424/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 10000000:01000000:3.0:1773941420.783415:0:12028:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-cliir 00000020:00000010:2.0:1773941420.783415:0:12100:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b89317400. 10000000:00000001:3.0:1773941420.783419:0:12028:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941420.783422:0:12028:0:(mgc_request.c:120:config_log_put()) Process leaving 00040000:00000010:2.0:1773941420.783422:0:12100:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98bb0900. 10000000:00000001:3.0:1773941420.783425:0:12028:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941420.783427:0:12028:0:(mgc_request.c:120:config_log_put()) Process leaving 00040000:00000010:2.0:1773941420.783428:0:12100:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b89316c00. 10000000:00000001:3.0:1773941420.783430:0:12028:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941420.783433:0:12028:0:(mgc_request.c:120:config_log_put()) Process leaving 00000020:00000010:2.0:1773941420.783434:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b89316600. 00000100:00000001:0.0:1773941420.783434:0:2352:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 10000000:00000001:3.0:1773941420.783435:0:12028:0:(mgc_request.c:117:config_log_put()) Process entered 00000100:00000001:0.0:1773941420.783437:0:2352:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941420.783438:0:12028:0:(mgc_request.c:120:config_log_put()) Process leaving 00000020:00000001:2.0:1773941420.783439:0:12100:0:(obd_class.h:696:obd_process_config()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941420.783442:0:2352:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000010:2.0:1773941420.783443:0:12100:0:(obd_mount.c:131:lustre_end_log()) kfreed 'lcfg': 176 at ffff8b5ba0b54a80. 02000000:00000001:0.0:1773941420.783445:0:2352:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 10000000:00000001:3.0:1773941420.783446:0:12028:0:(mgc_request.c:117:config_log_put()) Process entered 00000020:00000001:2.0:1773941420.783447:0:12100:0:(obd_mount.c:132:lustre_end_log()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941420.783448:0:2352:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941420.783449:0:12028:0:(mgc_request.c:120:config_log_put()) Process leaving 00000020:00000001:2.0:1773941420.783449:0:12100:0:(obd_mount.c:116:lustre_end_log()) Process entered 00000020:00000040:3.0:1773941420.783452:0:12028:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b885e2000 : new refcount 4 00000100:00100000:0.0:1773941420.783453:0:2352:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f10576f4-3fb6-413e-88ca-8953d457ee86:2352:1860112241143424:192.168.204.143@tcp:103:ldlm_bl.0 00000020:00000010:2.0:1773941420.783454:0:12100:0:(obd_mount.c:126:lustre_end_log()) kmalloced '(lcfg)': 168 at ffff8b5ba0b54a80. 10000000:00000010:3.0:1773941420.783458:0:12028:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 285 at ffff8b5b8809c800. 00000020:00000001:2.0:1773941420.783458:0:12100:0:(obd_class.h:681:obd_process_config()) Process entered 00000100:00000001:0.0:1773941420.783460:0:2352:0:(niobuf.c:865:ptl_send_rpc()) Process entered 10000000:00000001:3.0:1773941420.783462:0:12028:0:(mgc_request.c:147:config_log_put()) Process leaving 00000020:00000010:2.0:1773941420.783462:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b89316600. 02000000:00000001:0.0:1773941420.783464:0:2352:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 10000000:00000001:3.0:1773941420.783465:0:12028:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941420.783466:0:12100:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b89316c00. 02000000:00000001:0.0:1773941420.783467:0:2352:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.783470:0:12028:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000040:00000010:2.0:1773941420.783470:0:12100:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b89317400. 02000000:00000001:0.0:1773941420.783470:0:2352:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:3.0:1773941420.783474:0:12028:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941420.783476:0:12028:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000020:00000010:2.0:1773941420.783476:0:12100:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90f26000. 02000000:00000010:0.0:1773941420.783476:0:2352:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b88909e00. 00010000:00000001:3.0:1773941420.783478:0:12028:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000020:00000010:2.0:1773941420.783481:0:12100:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b89317000. 02000000:00000001:0.0:1773941420.783481:0:2352:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.783482:0:12028:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941420.783485:0:12028:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81484600 with handle 0xa33a5be14eae1c0c from hash 00040000:00000010:2.0:1773941420.783486:0:12100:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98bb0900. 00010000:00000001:3.0:1773941420.783488:0:12028:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00040000:00000010:2.0:1773941420.783490:0:12100:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b89316400. 00000400:00000200:0.0:1773941420.783490:0:2352:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941420.783491:0:12028:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941420.783493:0:12028:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941420.783495:0:12028:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 10000000:00000001:2.0:1773941420.783495:0:12100:0:(mgc_request.c:1932:mgc_process_config()) Process entered 00000400:00000010:0.0:1773941420.783496:0:2352:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00000001:3.0:1773941420.783497:0:12028:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 10000000:00000001:2.0:1773941420.783497:0:12100:0:(mgc_request_server.c:668:mgc_process_config_server()) Process entered 00010000:00000001:3.0:1773941420.783500:0:12028:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 10000000:00000001:2.0:1773941420.783500:0:12100:0:(mgc_request.c:493:config_log_end()) Process entered 00010000:00000001:3.0:1773941420.783502:0:12028:0:(ldlm_request.c:2242:ldlm_cli_cancel_list()) Process entered 10000000:00000001:2.0:1773941420.783502:0:12100:0:(mgc_request.c:159:config_log_find()) Process entered 10000000:00000001:2.0:1773941420.783504:0:12100:0:(mgc_request.c:105:config_log_get()) Process entered 00010000:00000001:3.0:1773941420.783505:0:12028:0:(ldlm_request.c:1396:ldlm_cli_cancel_req()) Process entered 10000000:00000040:2.0:1773941420.783505:0:12100:0:(mgc_request.c:108:config_log_get()) log params (ffff8b5b8809ca00) refs 2 00000400:00000010:0.0:1773941420.783505:0:2352:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 10000000:00000001:2.0:1773941420.783510:0:12100:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941420.783511:0:12028:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90276300. 10000000:00000001:2.0:1773941420.783512:0:12100:0:(mgc_request.c:177:config_log_find()) Process leaving (rc=18446615823973468672 : -128249736082944 : ffff8b5b8809ca00) 00000020:00000040:3.0:1773941420.783513:0:12028:0:(genops.c:1229:class_import_get()) import ffff8b5b88954000 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000200:0.0:1773941420.783514:0:2352:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241143424, portal 18 10000000:00000001:2.0:1773941420.783516:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 00000100:00000001:3.0:1773941420.783517:0:12028:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 10000000:00000040:2.0:1773941420.783519:0:12100:0:(mgc_request.c:123:config_log_put()) log params(ffff8b5b8809ca00) refs 2 02000000:00000001:3.0:1773941420.783520:0:12028:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1773941420.783520:0:2352:0:(client.c:3244:ptlrpc_request_addref()) Process entered 10000000:00000001:2.0:1773941420.783522:0:12100:0:(mgc_request.c:147:config_log_put()) Process leaving 02000000:00000001:3.0:1773941420.783523:0:12028:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.783524:0:2352:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615823923186304 : -128249786365312 : ffff8b5b850a8a80) 10000000:00000001:2.0:1773941420.783525:0:12100:0:(mgc_request.c:511:config_log_end()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941420.783528:0:12028:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5ba0ab9200. 10000000:00000001:2.0:1773941420.783530:0:12100:0:(mgc_request.c:2026:mgc_process_config()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.783532:0:12028:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941420.783532:0:12100:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b89316c00. 00000100:00000040:0.0:1773941420.783535:0:2352:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b850a8a80 x1860112241143424/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941420.783536:0:12028:0:(ldlm_request.c:1348:ldlm_cancel_pack()) Process entered 00000040:00000010:2.0:1773941420.783536:0:12100:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b89317400. 00010000:00010000:3.0:1773941420.783540:0:12028:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c0c lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85f82 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000020:00000010:2.0:1773941420.783540:0:12100:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90f26000. 00000020:00000010:2.0:1773941420.783545:0:12100:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b89317000. 00040000:00000010:2.0:1773941420.783549:0:12100:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98bb0900. 00010000:00010000:3.0:1773941420.783558:0:12028:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00000001:0.0:1773941420.783558:0:2352:0:(niobuf.c:69:ptl_send_buf()) Process entered 00040000:00000010:2.0:1773941420.783559:0:12100:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b89316400. 00010000:00000001:3.0:1773941420.783562:0:12028:0:(ldlm_request.c:1380:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1773941420.783563:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b89316600. 00000100:00000001:3.0:1773941420.783566:0:12028:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:2.0:1773941420.783567:0:12100:0:(obd_class.h:696:obd_process_config()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941420.783567:0:2352:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000100:00000001:3.0:1773941420.783569:0:12028:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941420.783571:0:12100:0:(obd_mount.c:131:lustre_end_log()) kfreed 'lcfg': 168 at ffff8b5ba0b54a80. 00000400:00000010:0.0:1773941420.783573:0:2352:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6cdd0. 00000020:00000001:2.0:1773941420.783574:0:12100:0:(obd_mount.c:132:lustre_end_log()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941420.783577:0:12028:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006af361f1] to pc [ptlrpcd_01_00+0] req@ffff8b5b90276300 x1860112241143552/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000080:00000001:2.0:1773941420.783577:0:12100:0:(llite_lib.c:983:client_common_put_super()) Process entered 00000080:00000001:2.0:1773941420.783579:0:12100:0:(vvp_dev.c:313:cl_sb_fini()) Process entered 00000020:00000001:2.0:1773941420.783583:0:12100:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000100:00000200:0.0:1773941420.783584:0:2352:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1860112241143424, offset 0 00000020:00000010:2.0:1773941420.783588:0:12100:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8b5b8509c2d0. 00000020:00000010:2.0:1773941420.783592:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b89316600. 00000400:00000200:0.0:1773941420.783595:0:2352:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000020:00000010:2.0:1773941420.783597:0:12100:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b89316400. 00000020:00000010:2.0:1773941420.783602:0:12100:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0166a20. 00000400:00000200:0.0:1773941420.783607:0:2352:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000008:00000010:2.0:1773941420.783608:0:12100:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f04e18. 00010000:00000001:3.0:1773941420.783612:0:12028:0:(ldlm_request.c:1464:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000010:2.0:1773941420.783614:0:12100:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b8a2fd780. 00010000:00000001:3.0:1773941420.783615:0:12028:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941420.783616:0:2352:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:3.0:1773941420.783617:0:12028:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00008000:00000010:2.0:1773941420.783618:0:12100:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d77880. 00010000:00000001:3.0:1773941420.783620:0:12028:0:(ldlm_request.c:2290:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.783622:0:12028:0:(ldlm_request.c:1689:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941420.783624:0:12028:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1773941420.783624:0:12100:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b85eeca80. 00000400:00000200:0.0:1773941420.783624:0:2352:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:3.0:1773941420.783630:0:12028:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c0c lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85f82 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00000010:2.0:1773941420.783630:0:12100:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b98a8b908. 00000400:00000200:0.0:1773941420.783632:0:2352:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000020:00000010:2.0:1773941420.783634:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b89317000. 00010000:00000001:3.0:1773941420.783639:0:12028:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000008:00000010:2.0:1773941420.783639:0:12100:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5b885b3e40. 00000400:00000200:0.0:1773941420.783639:0:2352:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:3.0:1773941420.783642:0:12028:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c0c lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85f82 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00020000:00000010:2.0:1773941420.783644:0:12100:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5b84cbc240. 00000400:00000200:0.0:1773941420.783646:0:2352:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00008000:00000010:2.0:1773941420.783650:0:12100:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5b82c37248. 00000400:00000200:0.0:1773941420.783654:0:2352:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [100:100:256:4294967295] 192.168.204.143@tcp LPNI seq info [100:100:8:4294967295] 00010000:00000040:3.0:1773941420.783655:0:12028:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962000 count: 0 00000080:00000010:2.0:1773941420.783655:0:12100:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b845f5780. 00000020:00000001:2.0:1773941420.783661:0:12100:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446615823923135192 : -128249786416424 : ffff8b5b8509c2d8) 00000400:00000200:0.0:1773941420.783662:0:2352:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00001000:2.0:1773941420.783667:0:12100:0:(cl_object.c:831:cl_env_get()) 1@ffff8b5b8509c2d0 00010000:00000001:3.0:1773941420.783668:0:12028:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941420.783670:0:12028:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000010:3.0:1773941420.783674:0:12028:0:(ldlm_lockd.c:2913:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff8b5ba0b543c0. 00010000:00000001:3.0:1773941420.783679:0:12028:0:(ldlm_lockd.c:2917:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941420.783679:0:2352:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000020:00000001:3.0:1773941420.783685:0:12028:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941420.783687:0:12028:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941420.783690:0:2352:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00010000:3.0:1773941420.783692:0:12028:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941420.783695:0:12028:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000800:00000010:0.0:1773941420.783696:0:2352:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101e00. 00000020:00000001:3.0:1773941420.783697:0:12028:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941420.783701:0:12028:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1773941420.783719:0:2352:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000020:00000001:3.0:1773941420.783721:0:12028:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941420.783724:0:12028:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941420.783726:0:12028:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1773941420.783727:0:2352:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941420.783732:0:2352:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101e00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941420.783733:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000080:00000010:3.0:1773941420.783739:0:2354:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b98a56150. 00000020:00000040:2.0:1773941420.783740:0:12100:0:(obd_config.c:1058:class_decref()) Decref lustre-clilmv-ffff8b5b885e5800 (ffff8b5b85510000) now 5 - lov 00000100:00000001:0.0:1773941420.783741:0:2352:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1773941420.783743:0:2354:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b89396910. 00000400:00000200:0.0:1773941420.783748:0:2352:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000080:00000010:3.0:1773941420.783751:0:2354:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b98a6c1e0. 00000100:00000001:0.0:1773941420.783751:0:2352:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294861147 00000100:00000001:3.0:1773941420.783755:0:2354:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941420.783757:0:2354:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1773941420.783758:0:2352:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294860384 00000100:00000001:0.0:1773941420.783764:0:2352:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294861147 00000100:00000040:3.0:1773941420.783769:0:2354:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90276300 x1860112241143552/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941420.783770:0:2352:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294860384 00000100:00000001:0.0:1773941420.783777:0:2352:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941420.783782:0:2354:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941420.783782:0:2352:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.783785:0:2354:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.783787:0:2352:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941420.783789:0:2354:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1773941420.783791:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1773941420.783792:0:2354:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1773941420.783792:0:12100:0:(lovsub_dev.c:57:lovsub_device_fini()) Process entered 02000000:00000001:3.0:1773941420.783795:0:2354:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1773941420.783795:0:12100:0:(lovsub_dev.c:61:lovsub_device_fini()) Process leaving (rc=18446615824122623360 : -128249586928256 : ffff8b5b90edb580) 00000100:00000001:0.0:1773941420.783797:0:2352:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1773941420.783800:0:2354:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f10576f4-3fb6-413e-88ca-8953d457ee86:2354:1860112241143552:192.168.204.143@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1773941420.783800:0:2352:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1773941420.783806:0:2354:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1773941420.783806:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941420.783808:0:2352:0:(client.c:1974:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1773941420.783811:0:2354:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1773941420.783811:0:2352:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941420.783814:0:2354:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941420.783816:0:2354:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941420.783821:0:2354:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b885b0400. 00000100:00000001:0.0:1773941420.783821:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941420.783825:0:2354:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941420.783830:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941420.783832:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1773941420.783834:0:2354:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f873c8. 00000080:00000010:0.0:1773941420.783838:0:2353:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b902a49d8. 00000400:00000010:3.0:1773941420.783840:0:2354:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a7f8. 00000080:00000010:0.0:1773941420.783844:0:2353:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b8502cb68. 00000100:00000200:3.0:1773941420.783845:0:2354:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241143552, portal 18 00020000:00000010:2.0:1773941420.783848:0:12100:0:(lovsub_dev.c:72:lovsub_device_free()) kfreed 'lsd': 72 at ffff8b5ba0166360. 00000100:00000001:3.0:1773941420.783849:0:2354:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000080:00000010:0.0:1773941420.783849:0:2353:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b902a20f0. 00000100:00000001:3.0:1773941420.783852:0:2354:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109626112 : -128249599925504 : ffff8b5b90276300) 00000100:00000001:0.0:1773941420.783854:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941420.783861:0:2354:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90276300 x1860112241143552/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.783868:0:2354:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941420.783872:0:2354:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941420.783875:0:2354:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a550. 00000100:00000200:3.0:1773941420.783878:0:2354:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1860112241143552, offset 0 00000400:00000200:3.0:1773941420.783884:0:2354:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941420.783887:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.783891:0:2354:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941420.783895:0:2354:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941420.783898:0:2354:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00020000:00000001:2.0:1773941420.783899:0:12100:0:(lovsub_dev.c:57:lovsub_device_fini()) Process entered 00000400:00000200:3.0:1773941420.783901:0:2354:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00020000:00000001:2.0:1773941420.783901:0:12100:0:(lovsub_dev.c:61:lovsub_device_fini()) Process leaving (rc=18446615824122622720 : -128249586928896 : ffff8b5b90edb300) 00000400:00000200:3.0:1773941420.783905:0:2354:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:3.0:1773941420.783914:0:2354:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [101:101:255:4294967295] 192.168.204.143@tcp LPNI seq info [101:101:7:4294967295] 00000400:00000200:3.0:1773941420.783919:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.783930:0:2354:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:3.0:1773941420.783936:0:2354:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:0.0:1773941420.783937:0:12106:0:(ldlm_lockd.c:2959:ldlm_bl_thread_main()) Process entered 00000800:00000010:3.0:1773941420.783940:0:2354:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69d00. 00010000:00000010:0.0:1773941420.783942:0:12106:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at ffff8b5ba039c5c0. 00000020:00000010:0.0:1773941420.783947:0:12106:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b88909400. 00000800:00000200:3.0:1773941420.783948:0:2354:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000020:00000010:0.0:1773941420.783952:0:12106:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b88909a00. 00000800:00000200:3.0:1773941420.783956:0:2354:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00020000:00000010:2.0:1773941420.783956:0:12100:0:(lovsub_dev.c:72:lovsub_device_free()) kfreed 'lsd': 72 at ffff8b5ba0166720. 00000020:00000010:0.0:1773941420.783958:0:12106:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5ba0b0c000. 00000800:00000200:3.0:1773941420.783959:0:2354:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69d00 type 1, nob 424 niov 1 nkiov 1 00020000:00000001:2.0:1773941420.783960:0:12100:0:(lov_dev.c:223:lov_device_fini()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.783964:0:2354:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941420.783964:0:12106:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b9829c670. 00000020:00000010:0.0:1773941420.783969:0:12106:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b88908000. 00000400:00000200:3.0:1773941420.783970:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941420.783974:0:2354:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294861147 80000000:00000010:0.0:1773941420.783974:0:12106:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b85ef6180. 00000100:00000001:3.0:1773941420.783978:0:2354:0:(niobuf.c:691:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, keep current deadline 4294861147 40000000:00000010:0.0:1773941420.783979:0:12106:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5ba039c680. 00000100:00000001:3.0:1773941420.783983:0:2354:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294861147 00100000:00000010:0.0:1773941420.783985:0:12106:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5ba0b0e000. 00000100:00000001:3.0:1773941420.783986:0:2354:0:(niobuf.c:691:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, keep current deadline 4294861147 00040000:00000010:0.0:1773941420.783989:0:12106:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b99101300. 00000100:00000001:3.0:1773941420.783990:0:2354:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941420.783994:0:2354:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:0.0:1773941420.783994:0:12106:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b84fef200. 00002000:00000010:0.0:1773941420.783998:0:12106:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0bf3c00. 00000100:00000001:3.0:1773941420.784000:0:2354:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941420.784002:0:12106:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5ba039c940. 00000100:00000001:3.0:1773941420.784003:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1773941420.784006:0:12100:0:(lu_object.c:1212:lu_site_fini()) kvfree 's->ls_bkts': 18432 at ffff8b5ba0c88000. 00000020:00000001:0.0:1773941420.784008:0:12106:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000100:00000001:3.0:1773941420.784009:0:2354:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:0.0:1773941420.784011:0:12106:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.784012:0:2354:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00010000:00010000:0.0:1773941420.784015:0:12106:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1773941420.784017:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941420.784019:0:2354:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1773941420.784019:0:12105:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 00000020:00000010:2.0:1773941420.784022:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 192 at ffff8b5ba0cfb900. 00010000:00000001:0.0:1773941420.784022:0:12105:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000001:3.0:1773941420.784024:0:2354:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941420.784026:0:12105:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1c13 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xb574482a8fd85f89 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941420.784029:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941420.784033:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5b83f09400. 00010000:00010000:0.0:1773941420.784040:0:12105:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138fa00 already unused, calling callback (ffffffffc1482c20) 00000080:00000010:2.0:1773941420.784041:0:12100:0:(vvp_dev.c:155:vvp_device_free()) kfreed 'site': 576 at ffff8b5b83f0bc00. 00000100:00000001:3.0:1773941420.784042:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000001:0.0:1773941420.784042:0:12105:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1773941420.784046:0:12105:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1c13 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xb574482a8fd85f89 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00000010:3.0:1773941420.784048:0:2355:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8b5b98a565e8. 00000080:00000010:3.0:1773941420.784054:0:2355:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8b5b89395488. 00010000:00000001:0.0:1773941420.784054:0:12105:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000080:00000010:3.0:1773941420.784061:0:2355:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8b5b98a6c0f0. 00000080:00000010:2.0:1773941420.784061:0:12100:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8b5b98a565e8. 00010000:00000001:0.0:1773941420.784062:0:12105:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941420.784064:0:12105:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1773941420.784066:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1773941420.784067:0:12100:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8b5b98a56150. 00000020:00000040:0.0:1773941420.784067:0:12105:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138fa00 refcount=3 00000020:00000001:0.0:1773941420.784072:0:12105:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859120640 : -128249850430976 : ffff8b5b8138fa00) 00000080:00000010:2.0:1773941420.784073:0:12100:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8b5b902a49d8. 00010000:00000001:0.0:1773941420.784076:0:12105:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859120640 : -128249850430976 : ffff8b5b8138fa00) 00000080:00000010:2.0:1773941420.784079:0:12100:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8b5b902a4dc8. 00010000:00000001:0.0:1773941420.784080:0:12105:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1773941420.784083:0:12105:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1c13 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1008400000000 nid: local remote: 0xb574482a8fd85f89 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00000010:2.0:1773941420.784086:0:12100:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8b5b85eec5e8. 10000000:00000001:0.0:1773941420.784090:0:12105:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000080:00000010:2.0:1773941420.784092:0:12100:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8b5b85eec150. 10000000:00010000:0.0:1773941420.784093:0:12105:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1c13 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1009400000000 nid: local remote: 0xb574482a8fd85f89 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00000010:2.0:1773941420.784097:0:12100:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8b5b85eecb28. 10000000:01000000:0.0:1773941420.784099:0:12105:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00000800:00000200:3.0:1773941420.784101:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000080:00000010:2.0:1773941420.784102:0:12100:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8b5b85eec348. 10000000:01000000:0.0:1773941420.784103:0:12105:0:(mgc_request.c:944:mgc_blocking_ast()) log params: stopping, won't requeue 00000800:00000010:3.0:1773941420.784105:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101e00. 10000000:00000001:0.0:1773941420.784106:0:12105:0:(mgc_request.c:117:config_log_put()) Process entered 00000080:00000010:2.0:1773941420.784108:0:12100:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8b5b98a6c0f0. 10000000:00000040:0.0:1773941420.784109:0:12105:0:(mgc_request.c:123:config_log_put()) log params(ffff8b5b8809ca00) refs 1 00000400:00000200:3.0:1773941420.784111:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:01000000:0.0:1773941420.784114:0:12105:0:(mgc_request.c:130:config_log_put()) dropping config log params 00000080:00000010:2.0:1773941420.784115:0:12100:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8b5b98a6c1e0. 10000000:00000001:0.0:1773941420.784115:0:12105:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941420.784118:0:12105:0:(mgc_request.c:120:config_log_put()) Process leaving 00000080:00000010:2.0:1773941420.784120:0:12100:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8b5b902a20f0. 10000000:00000001:0.0:1773941420.784121:0:12105:0:(mgc_request.c:117:config_log_put()) Process entered 00000400:00000200:3.0:1773941420.784124:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 10000000:00000001:0.0:1773941420.784124:0:12105:0:(mgc_request.c:120:config_log_put()) Process leaving 00000080:00000010:2.0:1773941420.784125:0:12100:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8b5b902a21e0. 10000000:00000001:0.0:1773941420.784126:0:12105:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941420.784128:0:12105:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941420.784130:0:12105:0:(mgc_request.c:117:config_log_put()) Process entered 00000080:00000010:2.0:1773941420.784131:0:12100:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8b5b845f5d20. 10000000:00000001:0.0:1773941420.784132:0:12105:0:(mgc_request.c:120:config_log_put()) Process leaving 00000400:00000200:3.0:1773941420.784133:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6cdd0 10000000:00000001:0.0:1773941420.784135:0:12105:0:(mgc_request.c:117:config_log_put()) Process entered 00000400:00000010:3.0:1773941420.784136:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6cdd0. 00000080:00000010:2.0:1773941420.784136:0:12100:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8b5b845f5c30. 10000000:00000001:0.0:1773941420.784137:0:12105:0:(mgc_request.c:120:config_log_put()) Process leaving 00000020:00000040:0.0:1773941420.784141:0:12105:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b885e2000 : new refcount 3 00000080:00000010:2.0:1773941420.784142:0:12100:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8b5b845f55a0. 00000100:00000001:3.0:1773941420.784143:0:2346:0:(events.c:34:request_out_callback()) Process entered 10000000:00000010:0.0:1773941420.784144:0:12105:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 279 at ffff8b5b8809ca00. 00000080:00000010:2.0:1773941420.784146:0:12100:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8b5b845f5690. 10000000:00000001:0.0:1773941420.784147:0:12105:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000200:3.0:1773941420.784149:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b850a8a80 x1860112241143424/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 10000000:00000001:0.0:1773941420.784150:0:12105:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1773941420.784151:0:12100:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8b5b89395488. 00010000:00000001:0.0:1773941420.784155:0:12105:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000080:00000010:2.0:1773941420.784157:0:12100:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8b5b89396910. 00010000:00000001:0.0:1773941420.784158:0:12105:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000080:00000010:2.0:1773941420.784161:0:12100:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8b5b8502cb68. 00010000:00000001:0.0:1773941420.784162:0:12105:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:3.0:1773941420.784165:0:2346:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000001:0.0:1773941420.784165:0:12105:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000080:00000010:2.0:1773941420.784167:0:12100:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8b5b8502c248. 00010000:00000001:0.0:1773941420.784169:0:12105:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:2.0:1773941420.784170:0:12100:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8b5b98a88920. 00000100:00000040:3.0:1773941420.784171:0:2346:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b850a8a80 x1860112241143424/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000080:00000010:2.0:1773941420.784174:0:12100:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8b5b98a8afe8. 00000020:00000040:0.0:1773941420.784179:0:12105:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138fa00 with handle 0xa33a5be14eae1c13 from hash 00000080:00000010:2.0:1773941420.784180:0:12100:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8b5b98a88b68. 00000100:00000001:3.0:1773941420.784181:0:2346:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.784184:0:2346:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:0.0:1773941420.784184:0:12105:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000080:00000010:2.0:1773941420.784185:0:12100:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8b5b98a88248. 00010000:00000001:0.0:1773941420.784186:0:12105:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000080:00000010:2.0:1773941420.784189:0:12100:0:(vvp_dev.c:159:vvp_device_free()) kfreed 'vdv': 72 at ffff8b5ba0166ea0. 00010000:00000001:0.0:1773941420.784189:0:12105:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941420.784192:0:12105:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1773941420.784194:0:12105:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00020000:00000001:2.0:1773941420.784196:0:12100:0:(lov_obd.c:754:lov_cleanup()) Process entered 00010000:00000001:0.0:1773941420.784196:0:12105:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1773941420.784200:0:12105:0:(ldlm_request.c:2242:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1773941420.784202:0:12105:0:(ldlm_request.c:1396:ldlm_cli_cancel_req()) Process entered 00000100:00000010:0.0:1773941420.784205:0:12105:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b850a8700. 00000020:00000040:0.0:1773941420.784207:0:12105:0:(genops.c:1229:class_import_get()) import ffff8b5b88954000 refcount=6 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941420.784209:0:12105:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941420.784210:0:12105:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941420.784212:0:12105:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941420.784216:0:12105:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b84fee600. 00000100:00000001:0.0:1773941420.784219:0:12105:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941420.784220:0:12105:0:(ldlm_request.c:1348:ldlm_cancel_pack()) Process entered 00010000:00010000:0.0:1773941420.784223:0:12105:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1c13 lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85f89 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000020:00000001:2.0:1773941420.784226:0:12100:0:(lu_tgt_pool.c:211:lu_tgt_pool_free()) Process entered 00010000:00010000:0.0:1773941420.784227:0:12105:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:0.0:1773941420.784230:0:12105:0:(ldlm_request.c:1380:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1773941420.784231:0:12100:0:(lu_tgt_pool.c:218:lu_tgt_pool_free()) kfreed 'op->op_array': 8 at ffff8b5b890ef778. 00000100:00000001:0.0:1773941420.784232:0:12105:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941420.784236:0:12105:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941420.784238:0:12100:0:(lu_tgt_pool.c:224:lu_tgt_pool_free()) Process leaving 00000100:00000040:0.0:1773941420.784242:0:12105:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000373a969d] to pc [ptlrpcd_00_01+1] req@ffff8b5b850a8700 x1860112241143680/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1773941420.784243:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941420.784248:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69d00. 00000400:00000200:3.0:1773941420.784255:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:0.0:1773941420.784262:0:12105:0:(ldlm_request.c:1464:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1773941420.784263:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00010000:00000001:0.0:1773941420.784281:0:12105:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941420.784282:0:2346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a550 00010000:00000001:0.0:1773941420.784284:0:12105:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941420.784287:0:12105:0:(ldlm_request.c:2290:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941420.784288:0:2346:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a550. 00010000:00000001:0.0:1773941420.784289:0:12105:0:(ldlm_request.c:1689:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941420.784293:0:12105:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.784294:0:2346:0:(events.c:34:request_out_callback()) Process entered 00010000:00010000:0.0:1773941420.784296:0:12105:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1c13 lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85f89 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941420.784302:0:2346:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90276300 x1860112241143552/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941420.784321:0:12105:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941420.784323:0:2346:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00010000:0.0:1773941420.784324:0:12105:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138fa00/0xa33a5be14eae1c13 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85f89 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941420.784329:0:2346:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90276300 x1860112241143552/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000040:0.0:1773941420.784331:0:12105:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec800 count: 0 00000100:00000001:3.0:1773941420.784340:0:2346:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.784342:0:2346:0:(events.c:68:request_out_callback()) Process leaving 00020000:00020000:2.0:1773941420.784343:0:12100:0:(lov_obd.c:783:lov_cleanup()) lustre-clilov-ffff8b5b885e5800: lov tgt 0 not cleaned! deathrow=0, lovrc=1 00010000:00000001:0.0:1773941420.784343:0:12105:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941420.784346:0:12105:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000010:0.0:1773941420.784348:0:12105:0:(ldlm_lockd.c:2913:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff8b5ba0b54480. 00010000:00000001:0.0:1773941420.784352:0:12105:0:(ldlm_lockd.c:2917:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941420.784354:0:12105:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941420.784355:0:12105:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941420.784359:0:12105:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1773941420.784361:0:12105:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941420.784363:0:12105:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941420.784365:0:12105:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1773941420.784368:0:12105:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941420.784369:0:12105:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941420.784371:0:12105:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1773941420.784376:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941420.784378:0:2353:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941420.784379:0:2353:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941420.784383:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b850a8700 x1860112241143680/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941420.784388:0:2353:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941420.784389:0:2353:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941420.784390:0:2353:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941420.784392:0:2353:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941420.784394:0:2353:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941420.784396:0:2353:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f10576f4-3fb6-413e-88ca-8953d457ee86:2353:1860112241143680:192.168.204.143@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1773941420.784398:0:2353:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941420.784400:0:2353:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941420.784402:0:2353:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941420.784407:0:2353:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941420.784409:0:2353:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b84feea00. 02000000:00000001:0.0:1773941420.784411:0:2353:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941420.784414:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941420.784416:0:2353:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93c08. 00000400:00000010:0.0:1773941420.784419:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6ce58. 00000100:00000200:0.0:1773941420.784422:0:2353:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241143680, portal 18 00000100:00000001:0.0:1773941420.784423:0:2353:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941420.784425:0:2353:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615823923185408 : -128249786366208 : ffff8b5b850a8700) 00000100:00000040:0.0:1773941420.784429:0:2353:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b850a8700 x1860112241143680/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941420.784434:0:2353:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941420.784436:0:2353:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941420.784438:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6ccc0. 00000100:00000200:0.0:1773941420.784440:0:2353:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1860112241143680, offset 0 00000400:00000200:0.0:1773941420.784443:0:2353:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941420.784447:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941420.784450:0:2353:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941420.784452:0:2353:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941420.784455:0:2353:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941420.784457:0:2353:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941420.784458:0:2353:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941420.784460:0:2353:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [102:102:256:4294967295] 192.168.204.143@tcp LPNI seq info [102:102:8:4294967295] 00000400:00000200:0.0:1773941420.784463:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941420.784469:0:2353:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941420.784472:0:2353:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941420.784475:0:2353:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101500. 00000800:00000200:0.0:1773941420.784478:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941420.784482:0:2353:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941420.784484:0:2353:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101500 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941420.784493:0:2353:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941420.784499:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941420.784501:0:2353:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294861147 00000100:00000001:0.0:1773941420.784503:0:2353:0:(niobuf.c:691:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, keep current deadline 4294861147 00000100:00000001:0.0:1773941420.784505:0:2353:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294861147 00000100:00000001:0.0:1773941420.784506:0:2353:0:(niobuf.c:691:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, keep current deadline 4294861147 00000100:00000001:0.0:1773941420.784508:0:2353:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941420.784511:0:2353:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.784513:0:2353:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.784516:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941420.784520:0:2353:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941420.784522:0:2353:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1773941420.784525:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941420.784526:0:2353:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941420.784528:0:2353:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.784530:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.784542:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941420.784544:0:2352:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941420.784546:0:2352:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941420.784549:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1773941420.784549:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1773941420.784555:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101500. 00000400:00000200:3.0:1773941420.784561:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:0.0:1773941420.784562:0:12106:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941420.784565:0:12106:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941420.784569:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00010000:00010000:0.0:1773941420.784570:0:12106:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1773941420.784574:0:12106:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000400:00000200:3.0:1773941420.784575:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6ccc0 00000020:00000001:0.0:1773941420.784577:0:12106:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941420.784580:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6ccc0. 00010000:00010000:0.0:1773941420.784580:0:12106:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1773941420.784586:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941420.784592:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b850a8700 x1860112241143680/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.784605:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941420.784607:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b850a8700 x1860112241143680/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.784612:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.784613:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941420.785615:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.785624:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941420.785626:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.785629:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.785634:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.785639:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941420.785645:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.785649:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.785653:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.785657:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 18 MB=0x69bc321602f00 00000400:00000200:3.0:1773941420.785661:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.785666:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.204.143@tcp of length 224/224 into md 0x86d [1] + 224 00000800:00000001:3.0:1773941420.785672:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.785679:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.785680:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941420.785683:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941420.785686:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941420.785691:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90276300 x1860112241143552/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941420.785700:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b90276300 x1860112241143552/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.785730:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941420.785735:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.785739:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941420.785788:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.785794:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941420.785801:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.785805:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.785809:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.785813:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941420.785819:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.785822:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.785824:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.785828:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 18 MB=0x69bc321602e80 00000400:00000200:3.0:1773941420.785832:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.785835:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.204.143@tcp of length 224/224 into md 0x865 [1] + 224 00000800:00000001:3.0:1773941420.785840:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.785864:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.785867:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941420.785871:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941420.785875:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941420.785879:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b850a8a80 x1860112241143424/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941420.785887:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b850a8a80 x1860112241143424/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.785900:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941420.785904:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.785906:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941420.785915:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941420.785917:0:2354:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941420.785920:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000001:3.0:1773941420.785921:0:2354:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1773941420.785923:0:2352:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941420.786020:0:2354:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.786022:0:2354:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f873c8. 00000400:00000200:3.0:1773941420.786026:0:2354:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a7f8 00000400:00000010:3.0:1773941420.786028:0:2354:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a7f8. 00000100:00000001:3.0:1773941420.786031:0:2354:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000001:0.0:1773941420.786032:0:2352:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941420.786034:0:2352:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000200:3.0:1773941420.786038:0:2354:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90276300 x1860112241143552/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941420.786038:0:2352:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941420.786042:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941420.786045:0:2352:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941420.786049:0:2352:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941420.786055:0:2352:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b850a8a80 x1860112241143424/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941420.786059:0:2354:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90276300 x1860112241143552/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941420.786064:0:2352:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b850a8a80 x1860112241143424/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.786083:0:2354:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941420.786084:0:2352:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941420.786088:0:2352:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941420.786089:0:2354:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941420.786090:0:2352:0:(client.c:1578:after_reply()) Process entered 00000100:00000001:3.0:1773941420.786093:0:2354:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941420.786094:0:2352:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 02000000:00000001:3.0:1773941420.786098:0:2354:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941420.786098:0:2352:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941420.786100:0:2352:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.786102:0:2354:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 02000000:00000001:0.0:1773941420.786102:0:2352:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.786106:0:2354:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941420.786108:0:2352:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b889544a0 time=1773941420 v=5 (0 0 0 0) 02000000:00000001:3.0:1773941420.786110:0:2354:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941420.786112:0:2352:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b88954410 time=6 v=5 (1 0 0 0) 00000100:00001000:3.0:1773941420.786116:0:2354:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b889544a0 time=0 v=5 (5 0 0 0) 00000100:00000001:0.0:1773941420.786116:0:2352:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941420.786117:0:2352:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941420.786121:0:2352:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00000100:00001000:3.0:1773941420.786124:0:2354:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b88954410 time=6 v=5 (1 0 0 0) 00010000:00000001:0.0:1773941420.786124:0:2352:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.786127:0:2352:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.786130:0:2354:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000040:0.0:1773941420.786133:0:2352:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b850a8a80 x1860112241143424/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.786134:0:2354:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.786140:0:2354:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00000100:00000001:0.0:1773941420.786141:0:2352:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1773941420.786143:0:2354:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.786144:0:2352:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941420.786146:0:2352:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941420.786148:0:2354:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941420.786152:0:2352:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b850a8a80 x1860112241143424/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941420.786158:0:2354:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90276300 x1860112241143552/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941420.786161:0:2352:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f10576f4-3fb6-413e-88ca-8953d457ee86:2352:1860112241143424:192.168.204.143@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1773941420.786169:0:2352:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941420.786172:0:2352:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941420.786177:0:2354:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1773941420.786177:0:2352:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b850a8a80 x1860112241143424/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.786184:0:2354:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941420.786187:0:2352:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941420.786188:0:2354:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1773941420.786190:0:2352:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941420.786193:0:2352:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b88909e00. 02000000:00000001:0.0:1773941420.786196:0:2352:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1773941420.786197:0:2354:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90276300 x1860112241143552/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1773941420.786198:0:2352:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941420.786204:0:2352:0:(genops.c:1242:class_import_put()) import ffff8b5b88954000 refcount=5 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941420.786208:0:2352:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00100000:3.0:1773941420.786211:0:2354:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f10576f4-3fb6-413e-88ca-8953d457ee86:2354:1860112241143552:192.168.204.143@tcp:103:ldlm_bl.0 02000000:00000010:0.0:1773941420.786211:0:2352:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b88908a00. 02000000:00000001:0.0:1773941420.786215:0:2352:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941420.786218:0:2352:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941420.786220:0:2352:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b850a8a80. 00000100:00000001:3.0:1773941420.786221:0:2354:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941420.786222:0:2352:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941420.786225:0:2352:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941420.786226:0:2354:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941420.786227:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941420.786232:0:2352:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1773941420.786234:0:2354:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90276300 x1860112241143552/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941420.786235:0:2352:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.786238:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941420.786241:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.786248:0:2354:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941420.786250:0:2354:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941420.786252:0:2354:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b885b0400. 02000000:00000001:3.0:1773941420.786255:0:2354:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941420.786257:0:2354:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941420.786258:0:2354:0:(genops.c:1242:class_import_put()) import ffff8b5b88954000 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941420.786277:0:2354:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941420.786279:0:2354:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5ba0ab9200. 02000000:00000001:3.0:1773941420.786282:0:2354:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941420.786284:0:2354:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941420.786286:0:2354:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90276300. 00000100:00000001:3.0:1773941420.786289:0:2354:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941420.786291:0:2354:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941420.786293:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941420.786297:0:2354:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941420.786298:0:2354:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.786300:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941420.786302:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1773941420.786310:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.786315:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941420.786316:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.786319:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.786323:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.786328:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941420.786333:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.786336:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.786340:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.786343:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 18 MB=0x69bc321602f80 00000400:00000200:3.0:1773941420.786347:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.786351:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.204.143@tcp of length 224/224 into md 0x87d [1] + 224 00000800:00000001:3.0:1773941420.786355:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.786361:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.786363:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941420.786367:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941420.786370:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941420.786374:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b850a8700 x1860112241143680/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941420.786383:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b850a8700 x1860112241143680/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.786397:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941420.786408:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.786412:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941420.786413:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941420.786415:0:2353:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941420.786418:0:2353:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941420.786419:0:2353:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941420.786421:0:2353:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93c08. 00000400:00000200:0.0:1773941420.786424:0:2353:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6ce58 00000400:00000010:0.0:1773941420.786425:0:2353:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6ce58. 00000100:00000001:0.0:1773941420.786426:0:2353:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941420.786429:0:2353:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b850a8700 x1860112241143680/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941420.786435:0:2353:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b850a8700 x1860112241143680/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941420.786438:0:2353:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941420.786440:0:2353:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941420.786441:0:2353:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941420.786442:0:2353:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941420.786443:0:2353:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941420.786444:0:2353:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941420.786445:0:2353:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941420.786447:0:2353:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b889544a0 time=0 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941420.786449:0:2353:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b88954410 time=6 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941420.786450:0:2353:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941420.786451:0:2353:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941420.786452:0:2353:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941420.786457:0:2353:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.786459:0:2353:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941420.786461:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b850a8700 x1860112241143680/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941420.786466:0:2353:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941420.786467:0:2353:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941420.786468:0:2353:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941420.786470:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b850a8700 x1860112241143680/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941420.786474:0:2353:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f10576f4-3fb6-413e-88ca-8953d457ee86:2353:1860112241143680:192.168.204.143@tcp:103:ldlm_bl.0 00000100:00000001:0.0:1773941420.786476:0:2353:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941420.786478:0:2353:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941420.786479:0:2353:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b850a8700 x1860112241143680/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941436 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941420.786483:0:2353:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941420.786484:0:2353:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941420.786485:0:2353:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b84feea00. 02000000:00000001:0.0:1773941420.786487:0:2353:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941420.786487:0:2353:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941420.786488:0:2353:0:(genops.c:1242:class_import_put()) import ffff8b5b88954000 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941420.786490:0:2353:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941420.786491:0:2353:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b84fee600. 02000000:00000001:0.0:1773941420.786492:0:2353:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941420.786493:0:2353:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941420.786494:0:2353:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b850a8700. 00000100:00000001:0.0:1773941420.786495:0:2353:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941420.786495:0:2353:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941420.786497:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941420.786499:0:2353:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941420.786500:0:2353:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.786501:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941420.786502:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1773941420.787900:0:12100:0:(lov_obd.c:562:lov_del_target()) Process entered 00020000:01000000:3.0:1773941420.787974:0:12100:0:(lov_obd.c:589:lov_del_target()) uuid: lustre-OST0000_UUID idx: 0 gen: 1 exp: ffff8b5b90f20800 active: 1 00020000:00000001:3.0:1773941420.787977:0:12100:0:(lov_obd.c:598:lov_del_target()) Process leaving (rc=0 : 0 : 0) 00020000:01000000:3.0:1773941420.787980:0:12100:0:(lov_obd.c:64:lov_tgts_putref()) destroying 1 lov targets 00000020:00000001:3.0:1773941420.787982:0:12100:0:(lu_tgt_pool.c:163:lu_tgt_pool_remove()) Process entered 00000020:00000001:3.0:1773941420.787983:0:12100:0:(lu_tgt_pool.c:180:lu_tgt_pool_remove()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00020000:00000001:3.0:1773941420.787985:0:12100:0:(lu_tgt_descs.c:164:lu_qos_del_tgt()) Process entered 00020000:00000001:3.0:1773941420.787987:0:12100:0:(lu_tgt_descs.c:169:lu_qos_del_tgt()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00020000:00000001:3.0:1773941420.787989:0:12100:0:(lu_tgt_descs.c:186:lu_qos_del_tgt()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00020000:01000000:3.0:1773941420.787990:0:12100:0:(lov_obd.c:611:__lov_del_obd()) Removing tgt lustre-OST0000_UUID : lustre-OST0000-osc-ffff8b5b885e5800 00020000:00000001:3.0:1773941420.787992:0:12100:0:(lov_obd.c:245:lov_disconnect_obd()) Process entered 00020000:01000000:3.0:1773941420.787993:0:12100:0:(lov_obd.c:248:lov_disconnect_obd()) lustre-clilov-ffff8b5b885e5800: disconnecting target lustre-OST0000-osc-ffff8b5b885e5800 00020000:00000001:3.0:1773941420.787995:0:12100:0:(obd_class.h:1347:obd_register_observer()) Process entered 00020000:00000001:3.0:1773941420.787997:0:12100:0:(obd_class.h:1360:obd_register_observer()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1773941420.787998:0:12100:0:(obd_class.h:902:obd_disconnect()) Process entered 00010000:00000001:3.0:1773941420.788001:0:12100:0:(ldlm_lib.c:759:client_disconnect_export()) Process entered 00010000:00000040:3.0:1773941420.788003:0:12100:0:(ldlm_lib.c:772:client_disconnect_export()) disconnect lustre-OST0000-osc-ffff8b5b885e5800 - 1 00000100:00000001:3.0:1773941420.788005:0:12100:0:(pinger.c:388:ptlrpc_pinger_del_import()) Process entered 00000100:00080000:3.0:1773941420.788007:0:12100:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 637f784d-29a5-48a3-b831-1c4c7c7a2a54->lustre-OST0000_UUID 00000020:00000001:3.0:1773941420.788008:0:12100:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941420.788009:0:12100:0:(genops.c:1242:class_import_put()) import ffff8b5b88957800 refcount=1 obd=lustre-OST0000-osc-ffff8b5b885e5800 00000020:00000001:3.0:1773941420.788010:0:12100:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941420.788011:0:12100:0:(pinger.c:401:ptlrpc_pinger_del_import()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.788013:0:12100:0:(ldlm_request.c:2367:ldlm_cli_cancel_unused()) Process entered 00000001:00000001:3.0:1773941420.788015:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.788017:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.788044:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.788046:0:12100:0:(ldlm_request.c:2379:ldlm_cli_cancel_unused()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.788048:0:12100:0:(ldlm_resource.c:1268:ldlm_namespace_free_prior()) Process entered 00010000:00000001:3.0:1773941420.788050:0:12100:0:(ldlm_resource.c:1207:__ldlm_namespace_free()) Process entered 00000001:00000001:3.0:1773941420.788051:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.788052:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.788074:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.788075:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.788081:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000080:00000010:2.1F:1773941420.788103:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8b5b98068140. 00000001:00000001:3.0:1773941420.788105:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.788107:0:12100:0:(ldlm_resource.c:1251:__ldlm_namespace_free()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.788109:0:12100:0:(ldlm_resource.c:1293:ldlm_namespace_free_prior()) Process leaving 00000100:00000001:3.0:1773941420.788111:0:12100:0:(import.c:1924:ptlrpc_disconnect_import()) Process entered 00000100:00000001:3.0:1773941420.788115:0:12100:0:(import.c:1858:ptlrpc_disconnect_import_async()) Process entered 00000100:00000001:3.0:1773941420.788116:0:12100:0:(import.c:1750:ptlrpc_disconnect_prep_req()) Process entered 00000100:00000010:3.0:1773941420.788119:0:12100:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90276300. 00000020:00000040:3.0:1773941420.788121:0:12100:0:(genops.c:1229:class_import_get()) import ffff8b5b88957800 refcount=2 obd=lustre-OST0000-osc-ffff8b5b885e5800 00000100:00000001:3.0:1773941420.788124:0:12100:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941420.788125:0:12100:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 00000080:00000010:2.1:1773941420.788125:0:26:0:(super25.c:59:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8b5b83538840. 02000000:00000001:3.0:1773941420.788129:0:12100:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941420.788134:0:12100:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b98a69d00. 00000100:00000001:3.0:1773941420.788137:0:12100:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.788139:0:12100:0:(import.c:1788:ptlrpc_disconnect_prep_req()) Process leaving (rc=18446615824109626112 : -128249599925504 : ffff8b5b90276300) 00000100:00080000:3.0:1773941420.788141:0:12100:0:(import.c:65:import_set_state_nolock()) ffff8b5b88957800 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:00000001:3.0:1773941420.788143:0:12100:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941420.788145:0:12100:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941420.788149:0:12100:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006af361f1] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90276300 x1860112241143808/t0(0) o9->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.788163:0:12100:0:(import.c:1904:ptlrpc_disconnect_import_async()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.788174:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941420.788181:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941420.788182:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941420.788189:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90276300 x1860112241143808/t0(0) o9->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941420.788197:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941420.788199:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941420.788201:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941420.788203:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941420.788206:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941420.788209:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:637f784d-29a5-48a3-b831-1c4c7c7a2a54:2351:1860112241143808:192.168.204.143@tcp:9:umount.0 00000100:00000001:2.0:1773941420.788213:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941420.788216:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941420.788218:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941420.788220:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941420.788224:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b89317400. 02000000:00000001:2.0:1773941420.788226:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941420.788230:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941420.788234:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af97e8. 00000400:00000010:2.0:1773941420.788238:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f344c8. 00000100:00000200:2.0:1773941420.788242:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241143808, portal 4 00000100:00000001:2.0:1773941420.788244:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941420.788245:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109626112 : -128249599925504 : ffff8b5b90276300) 00000020:00000010:2.0:1773941420.788252:0:2351:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff8b5ba0c76000. 00000100:00000040:2.0:1773941420.788258:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90276300 x1860112241143808/t0(0) o9->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941420.788265:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941420.788281:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941420.788284:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34990. 00000100:00000200:2.0:1773941420.788287:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1860112241143808, offset 0 00000400:00000200:2.0:1773941420.788293:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941420.788298:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941420.788302:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941420.788313:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941420.788317:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941420.788319:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941420.788322:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941420.788325:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [103:103:256:4294967295] 192.168.204.143@tcp LPNI seq info [103:103:8:4294967295] 00000400:00000200:2.0:1773941420.788330:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941420.788337:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941420.788342:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941420.788346:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0900. 00000800:00000200:2.0:1773941420.788351:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941420.788356:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941420.788359:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0900 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1773941420.788371:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941420.788375:0:2351:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff8b5b885e5800: using a timeout of 2091 usecs (3 jiffies) 00000400:00000200:2.0:1773941420.788379:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941420.788381:0:2351:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff8b5b885e5800: PM QoS new deadline estimation for cpu 2 is 4294858583 00000100:00000001:2.0:1773941420.788385:0:2351:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff8b5b885e5800: PM QoS request active for cpu 2, keep current deadline 4294861147 00000100:00000001:2.0:1773941420.788387:0:2351:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff8b5b885e5800: PM QoS new deadline estimation for cpu 3 is 4294858583 00000100:00000001:2.0:1773941420.788390:0:2351:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff8b5b885e5800: PM QoS request active for cpu 3, keep current deadline 4294861147 00000100:00000001:2.0:1773941420.788392:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941420.788395:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.788398:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.788400:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941420.788404:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941420.788406:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941420.788408:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941420.788410:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941420.788412:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.788413:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941420.788421:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941420.788424:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0900. 00000400:00000200:3.0:1773941420.788427:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941420.788430:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941420.788433:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34990 00000400:00000010:3.0:1773941420.788434:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34990. 00000100:00000001:3.0:1773941420.788436:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941420.788439:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90276300 x1860112241143808/t0(0) o9->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.788444:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941420.788446:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90276300 x1860112241143808/t0(0) o9->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941430 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.788459:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.788460:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000010:3.1F:1773941420.788745:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962000. 00010000:00000010:3.1:1773941420.788755:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81484600. 00000800:00000001:3.0:1773941420.789563:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.789569:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941420.789570:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.789573:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.789576:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.789580:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941420.789584:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.789586:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.789589:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.789593:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 4 MB=0x69bc321603000 00000400:00000200:3.0:1773941420.789596:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.789599:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.143@tcp of length 224/224 into md 0x885 [1] + 224 00000800:00000001:3.0:1773941420.789603:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.789608:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.789609:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941420.789613:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941420.789617:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941420.789622:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90276300 x1860112241143808/t0(0) o9->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941420.789632:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b90276300 x1860112241143808/t0(0) o9->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.789645:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941420.789651:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.789655:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941420.789664:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941420.789667:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941420.789676:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941420.789678:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941420.789681:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af97e8. 00000400:00000200:2.0:1773941420.789684:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f344c8 00000400:00000010:2.0:1773941420.789686:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f344c8. 00000100:00000001:2.0:1773941420.789689:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941420.789695:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90276300 x1860112241143808/t0(0) o9->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941420.789730:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90276300 x1860112241143808/t0(0) o9->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941420.789737:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941420.789739:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941420.789741:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941420.789742:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941420.789743:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941420.789745:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941420.789747:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941420.789751:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b88957c58 time=1 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941420.789754:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b88957c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941420.789757:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941420.789758:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.789760:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1773941420.789765:0:2351:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=92324) req@ffff8b5b90276300 x1860112241143808/t0(0) o9->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941420.789770:0:2351:0:(ldlm_request.c:1535:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.789772:0:2351:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1773941420.789774:0:2351:0:(client.c:3077:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b5b885e5800: skip recheck: last_committed 0 00000100:00000001:2.0:1773941420.789775:0:2351:0:(client.c:3078:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1773941420.789777:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941420.789780:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90276300 x1860112241143808/t0(0) o9->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941420.789784:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941420.789785:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941420.789791:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941420.789793:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b88957800 lustre-OST0000_UUID: changing import state from CONNECTING to CLOSED 00000100:00000001:2.0:1773941420.789795:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 00000008:00000001:2.0:1773941420.789798:0:2351:0:(osc_request.c:3880:osc_import_event()) Process entered 00000008:00000001:2.0:1773941420.789799:0:2351:0:(osc_request.c:3947:osc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.789801:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941420.789802:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 00000008:00000001:2.0:1773941420.789803:0:2351:0:(osc_request.c:3880:osc_import_event()) Process entered 00000008:00000001:2.0:1773941420.789805:0:2351:0:(osc_request.c:3947:osc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.789806:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000040:2.0:1773941420.789815:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90276300 x1860112241143808/t0(0) o9->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941420.789820:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:637f784d-29a5-48a3-b831-1c4c7c7a2a54:2351:1860112241143808:192.168.204.143@tcp:9:umount.0 00000100:00000001:3.0:1773941420.789823:0:12100:0:(import.c:1967:ptlrpc_disconnect_import()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.789825:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941420.789828:0:12100:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00000001:2.0:1773941420.789829:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1773941420.789832:0:12100:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0000_UUID INVALID 00000100:00000040:2.0:1773941420.789835:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90276300 x1860112241143808/t0(0) o9->lustre-OST0000-osc-ffff8b5b885e5800@192.168.204.143@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.789836:0:12100:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:3.0:1773941420.789839:0:12100:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00000001:2.0:1773941420.789843:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:3.0:1773941420.789847:0:12100:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b5b885e5800: committing for last_committed 0 gen 2 00000100:00000040:3.0:1773941420.789849:0:12100:0:(client.c:3191:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b5b885e5800: normal: skip=0 replay=0/0 committed=0/0 00000100:00000001:3.0:1773941420.789851:0:12100:0:(client.c:3192:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1773941420.789852:0:12100:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941420.789853:0:12100:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941420.789855:0:12100:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941420.789855:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000008:00000001:3.0:1773941420.789856:0:12100:0:(osc_request.c:3880:osc_import_event()) Process entered 00000008:00000001:3.0:1773941420.789860:0:12100:0:(osc_request.c:3947:osc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941420.789860:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b89317400. 00000100:00000001:3.0:1773941420.789865:0:12100:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:2.0:1773941420.789866:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:3.0:1773941420.789867:0:12100:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000001:2.0:1773941420.789868:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941420.789870:0:12100:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000040:2.0:1773941420.789871:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b88957800 refcount=1 obd=lustre-OST0000-osc-ffff8b5b885e5800 00000008:00000001:3.0:1773941420.789872:0:12100:0:(osc_request.c:3880:osc_import_event()) Process entered 00000020:00000001:2.0:1773941420.789875:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:3.0:1773941420.789876:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941420.789878:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b98a69d00. 00000001:00000001:3.0:1773941420.789879:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941420.789883:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941420.789884:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941420.789886:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90276300. 00000100:00000001:2.0:1773941420.789888:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941420.789889:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941420.789890:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941420.789893:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941420.789895:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.789896:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000001:00000001:3.0:1773941420.789903:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.789903:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.789906:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.789907:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.789929:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.789930:0:12100:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1773941420.789942:0:12100:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8b5b850f85a0. 00000020:00000010:3.0:1773941420.789946:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b885b0400. 00000020:00000010:3.0:1773941420.789948:0:12100:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b885b1600. 00000020:00000010:3.0:1773941420.789950:0:12100:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9a20. 00000008:00000010:3.0:1773941420.789954:0:12100:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f0d940. 00020000:00000010:3.0:1773941420.789957:0:12100:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b89382c00. 00008000:00000010:3.0:1773941420.789960:0:12100:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b98a1e6d8. 00000020:00000010:3.0:1773941420.789963:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b885b0600. 00000008:00000010:3.0:1773941420.789966:0:12100:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5ba0c32c60. 00020000:00000010:3.0:1773941420.789968:0:12100:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5ba0c346c0. 00008000:00000010:3.0:1773941420.789971:0:12100:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5b98a6ba30. 00000020:00000001:3.0:1773941420.789973:0:12100:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446615823923512744 : -128249786038872 : ffff8b5b850f85a8) 00000020:00001000:3.0:1773941420.789975:0:12100:0:(cl_object.c:831:cl_env_get()) 1@ffff8b5b850f85a0 00000008:00000001:3.0:1773941420.789977:0:12100:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1773941420.789979:0:12100:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1773941420.789981:0:12100:0:(osc_cache.c:2316:osc_next_obj()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1773941420.789982:0:12100:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000001:00000001:3.0:1773941420.789986:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.789987:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.790005:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:3.0:1773941420.790007:0:12100:0:(cl_object.c:925:cl_env_put()) 1@ffff8b5b850f85a0 00000020:00000010:3.0:1773941420.790009:0:12100:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b885b1600. 00000020:00000010:3.0:1773941420.790012:0:12100:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9a20. 00000008:00000010:3.0:1773941420.790015:0:12100:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5ba0f0d940. 00020000:00000010:3.0:1773941420.790018:0:12100:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b89382c00. 00008000:00000010:3.0:1773941420.790021:0:12100:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b98a1e6d8. 00000020:00000010:3.0:1773941420.790023:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b885b0400. 00000008:00000010:3.0:1773941420.790026:0:12100:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8b5ba0c32c60. 00020000:00000010:3.0:1773941420.790029:0:12100:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5ba0c346c0. 00008000:00000010:3.0:1773941420.790032:0:12100:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8b5b98a6ba30. 00000020:00000010:3.0:1773941420.790034:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b885b0600. 00000020:00000010:3.0:1773941420.790037:0:12100:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8b5b850f85a0. 00000001:00000001:3.0:1773941420.790039:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.790039:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.790058:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.790059:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.790060:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.790079:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1773941420.790080:0:12100:0:(osc_request.c:3947:osc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.790081:0:12100:0:(obd_class.h:1226:obd_import_event()) Process leaving 00010000:00000001:3.0:1773941420.790084:0:12100:0:(ldlm_lib.c:818:client_disconnect_export()) Process leaving 00000020:00000001:3.0:1773941420.790085:0:12100:0:(genops.c:1462:class_disconnect()) Process entered 00000020:00000080:3.0:1773941420.790087:0:12100:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0xa33a5be14eae1c2f 00000020:00000040:3.0:1773941420.790089:0:12100:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b90f20800 with handle 0xa33a5be14eae1c2f from hash 00000020:00000040:3.0:1773941420.790098:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b90f20800 : new refcount 2 00000020:00000001:3.0:1773941420.790100:0:12100:0:(genops.c:1934:obd_stale_export_put()) Process entered 00000020:00000040:3.0:1773941420.790101:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b90f20800 : new refcount 1 00000020:00000001:3.0:1773941420.790103:0:12100:0:(genops.c:1959:obd_stale_export_put()) Process leaving 00000020:00000040:3.0:1773941420.790104:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b90f20800 : new refcount 0 00000020:00000080:3.0:1773941420.790105:0:12100:0:(genops.c:904:class_export_put()) final put ffff8b5b90f20800/LOV_OSC_UUID 00000020:00000001:3.0:1773941420.790107:0:12100:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.790116:0:12100:0:(genops.c:1493:class_disconnect()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.790118:0:12100:0:(ldlm_lib.c:831:client_disconnect_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941420.790137:0:37:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:0.0:1773941420.790148:0:37:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b90f20800/LOV_OSC_UUID for lustre-OST0000-osc-ffff8b5b885e5800 00020000:00000001:3.0:1773941420.790150:0:12100:0:(obd_class.h:914:obd_disconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941420.790153:0:37:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 8 to 192.168.204.143@tcp 00020000:00000001:3.0:1773941420.790154:0:12100:0:(lov_obd.c:280:lov_disconnect_obd()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1773941420.790158:0:12100:0:(lov_obd.c:616:__lov_del_obd()) kfreed 'tgt': 368 at ffff8b5b850b6600. 00000020:00000001:0.0:1773941420.790159:0:37:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:0.0:1773941420.790162:0:37:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.790163:0:12100:0:(obd_config.c:2489:class_manual_cleanup()) Process entered 00000020:00000040:0.0:1773941420.790164:0:37:0:(obd_config.c:1058:class_decref()) Decref lustre-OST0000-osc-ffff8b5b885e5800 (ffff8b5ba07399b8) now 5 - export 00000020:01000000:3.0:1773941420.790165:0:12100:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-OST0000-osc-ffff8b5b885e5800 (flags='') 00000020:00000010:0.0:1773941420.790170:0:37:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff8b5b90f20800. 00000020:00000010:3.0:1773941420.790172:0:12100:0:(obd_config.c:2506:class_manual_cleanup()) kmalloced '(lcfg)': 88 at ffff8b5ba0bc9a20. 00000020:00000080:3.0:1773941420.790175:0:12100:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000001:3.0:1773941420.790179:0:12100:0:(obd_config.c:936:class_cleanup()) Process entered 00000020:00000001:0.0:1773941420.790179:0:37:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000080:3.0:1773941420.790184:0:12100:0:(obd_config.c:991:class_cleanup()) lustre-OST0000-osc-ffff8b5b885e5800: forcing exports to disconnect: 0/2 00000020:00000001:3.0:1773941420.790187:0:12100:0:(genops.c:1562:class_disconnect_exports()) Process entered 00000020:00080000:3.0:1773941420.790188:0:12100:0:(genops.c:1577:class_disconnect_exports()) OBD device 4 (ffff8b5ba07399b8) has no exports 00000020:00000001:3.0:1773941420.790190:0:12100:0:(genops.c:1578:class_disconnect_exports()) Process leaving 00000020:00000001:3.0:1773941420.790193:0:12100:0:(obd_class.h:604:obd_precleanup()) Process entered 00000020:00000010:3.0:1773941420.790195:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b885b0600. 00000020:00000010:3.0:1773941420.790197:0:12100:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b885b0400. 00000020:00000010:3.0:1773941420.790198:0:12100:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc91e0. 00000008:00000010:3.0:1773941420.790200:0:12100:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f0d940. 00020000:00000010:3.0:1773941420.790202:0:12100:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b89382c00. 00008000:00000010:3.0:1773941420.790204:0:12100:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b98a1e6d8. 00000020:00000010:3.0:1773941420.790207:0:12100:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068400. 00000008:00000001:3.0:1773941420.790209:0:12100:0:(osc_dev.c:163:osc_device_fini()) Process entered 00000008:00000001:3.0:1773941420.790211:0:12100:0:(osc_request.c:4123:osc_precleanup_common()) Process entered 00000008:00000001:3.0:1773941420.790215:0:12100:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 00000008:01000000:3.0:1773941420.790216:0:12100:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-OST0000-osc-ffff8b5b885e5800: client import never connected 00000100:00000001:3.0:1773941420.790218:0:12100:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941420.790219:0:12100:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0000_UUID INVALID 00000100:00000001:3.0:1773941420.790220:0:12100:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:3.0:1773941420.790221:0:12100:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00100000:3.0:1773941420.790222:0:12100:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b5b885e5800: committing for last_committed 0 gen 3 00000100:00000040:3.0:1773941420.790223:0:12100:0:(client.c:3191:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b5b885e5800: normal: skip=0 replay=0/0 committed=0/0 00000100:00000001:3.0:1773941420.790225:0:12100:0:(client.c:3192:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1773941420.790225:0:12100:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941420.790226:0:12100:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941420.790227:0:12100:0:(obd_class.h:1215:obd_import_event()) Process entered 00000008:00000001:3.0:1773941420.790228:0:12100:0:(osc_request.c:3880:osc_import_event()) Process entered 00000008:00000001:3.0:1773941420.790235:0:12100:0:(osc_request.c:3947:osc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.790236:0:12100:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00100000:3.0:1773941420.790237:0:12100:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941420.790239:0:12100:0:(obd_class.h:1215:obd_import_event()) Process entered 00000008:00000001:3.0:1773941420.790240:0:12100:0:(osc_request.c:3880:osc_import_event()) Process entered 00000001:00000001:3.0:1773941420.790241:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.790242:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.790263:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.790264:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.790265:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.790300:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.790302:0:12100:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1773941420.790304:0:12100:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8b5b850f85a0. 00000020:00000010:3.0:1773941420.790307:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b885b1600. 00000020:00000010:3.0:1773941420.790309:0:12100:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab9200. 00000020:00000010:3.0:1773941420.790311:0:12100:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9f00. 00000008:00000010:3.0:1773941420.790314:0:12100:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f0ce18. 00020000:00000010:3.0:1773941420.790315:0:12100:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b89382a80. 00008000:00000010:3.0:1773941420.790318:0:12100:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b98a1d1a8. 00000020:00000010:3.0:1773941420.790321:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84e8e400. 00000008:00000010:3.0:1773941420.790322:0:12100:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8b5ba0c32c60. 00020000:00000010:3.0:1773941420.790324:0:12100:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8b5ba0c346c0. 00008000:00000010:3.0:1773941420.790325:0:12100:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5b98a6ba30. 00000020:00000001:3.0:1773941420.790327:0:12100:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446615823923512744 : -128249786038872 : ffff8b5b850f85a8) 00000020:00001000:3.0:1773941420.790328:0:12100:0:(cl_object.c:831:cl_env_get()) 1@ffff8b5b850f85a0 00000008:00000001:3.0:1773941420.790329:0:12100:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1773941420.790330:0:12100:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1773941420.790331:0:12100:0:(osc_cache.c:2316:osc_next_obj()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1773941420.790332:0:12100:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000001:00000001:3.0:1773941420.790333:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.790334:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.790354:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:3.0:1773941420.790356:0:12100:0:(cl_object.c:925:cl_env_put()) 1@ffff8b5b850f85a0 00000020:00000010:3.0:1773941420.790357:0:12100:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab9200. 00000020:00000010:3.0:1773941420.790360:0:12100:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9f00. 00000008:00000010:3.0:1773941420.790362:0:12100:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5ba0f0ce18. 00020000:00000010:3.0:1773941420.790365:0:12100:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b89382a80. 00008000:00000010:3.0:1773941420.790372:0:12100:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b98a1d1a8. 00000020:00000010:3.0:1773941420.790374:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b885b1600. 00000008:00000010:3.0:1773941420.790375:0:12100:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8b5ba0c32c60. 00020000:00000010:3.0:1773941420.790378:0:12100:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5ba0c346c0. 00008000:00000010:3.0:1773941420.790379:0:12100:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8b5b98a6ba30. 00000020:00000010:3.0:1773941420.790381:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84e8e400. 00000020:00000010:3.0:1773941420.790382:0:12100:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8b5b850f85a0. 00000001:00000001:3.0:1773941420.790384:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.790385:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.790405:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.790406:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.790407:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.790429:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1773941420.790430:0:12100:0:(osc_request.c:3947:osc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.790432:0:12100:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:3.0:1773941420.790434:0:12100:0:(genops.c:1229:class_import_get()) import ffff8b5b88957800 refcount=2 obd=lustre-OST0000-osc-ffff8b5b885e5800 00000020:00000001:3.0:1773941420.790435:0:12100:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941420.790436:0:12100:0:(genops.c:1242:class_import_put()) import ffff8b5b88957800 refcount=1 obd=lustre-OST0000-osc-ffff8b5b885e5800 00000020:00000001:3.0:1773941420.790437:0:12100:0:(genops.c:1249:class_import_put()) Process leaving 00000020:00000001:3.0:1773941420.790439:0:12100:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941420.790440:0:12100:0:(genops.c:1242:class_import_put()) import ffff8b5b88957800 refcount=0 obd=lustre-OST0000-osc-ffff8b5b885e5800 00000020:00000040:3.0:1773941420.790441:0:12100:0:(genops.c:1245:class_import_put()) final put import ffff8b5b88957800 00000020:00000001:3.0:1773941420.790449:0:12100:0:(genops.c:1249:class_import_put()) Process leaving 00000008:00000001:3.0:1773941420.790450:0:12100:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 00000008:00000001:3.0:1773941420.790451:0:12100:0:(osc_request.c:4138:osc_precleanup_common()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941420.790505:0:37:0:(genops.c:1200:obd_zombie_import_free()) Process entered 00000020:00000080:0.0:1773941420.790508:0:37:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff8b5b88957800 for lustre-OST0000-osc-ffff8b5b885e5800 00000020:00000040:0.0:1773941420.790511:0:37:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 7 to 192.168.204.143@tcp 00000020:00000040:0.0:1773941420.790514:0:37:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 6 to 192.168.204.143@tcp 00000020:00000010:0.0:1773941420.790516:0:37:0:(genops.c:1213:obd_zombie_import_free()) kfreed 'imp_conn': 88 at ffff8b5b880f9d80. 00000020:00000040:0.0:1773941420.790520:0:37:0:(obd_config.c:1058:class_decref()) Decref lustre-OST0000-osc-ffff8b5b885e5800 (ffff8b5ba07399b8) now 4 - import 00000020:00000010:0.0:1773941420.790522:0:37:0:(genops.c:1220:obd_zombie_import_free()) kfreed 'imp': 1712 at ffff8b5b88957800. 00000020:00000001:0.0:1773941420.790526:0:37:0:(genops.c:1221:obd_zombie_import_free()) Process leaving 00000020:00000010:3.0:1773941420.790691:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff8b5b85e56000. 00000020:00000010:3.0:1773941420.790694:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff8b5ba0c76000. 00000020:00000010:3.0:1773941420.790697:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 3680 at ffff8b5b90f37000. 00000020:00000010:3.0:1773941420.790717:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5ba0737000. 00000008:00000001:3.0:1773941420.790720:0:12100:0:(osc_dev.c:168:osc_device_fini()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.790723:0:12100:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068400. 00000020:00000010:3.0:1773941420.790733:0:12100:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b885b0400. 00000020:00000010:3.0:1773941420.790737:0:12100:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc91e0. 00000008:00000010:3.0:1773941420.790740:0:12100:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5ba0f0d940. 00020000:00000010:3.0:1773941420.790745:0:12100:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b89382c00. 00010000:00000010:0.1:1773941420.790748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81484a00. 00008000:00000010:3.0:1773941420.790753:0:12100:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b98a1e6d8. 00000020:00000010:3.0:1773941420.790757:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b885b0600. 00000020:00000001:3.0:1773941420.790761:0:12100:0:(obd_class.h:623:obd_precleanup()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:0.1:1773941420.790766:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec800. 00010000:00000010:0.1:1773941420.790775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138fa00. 00000001:00000001:3.0:1773941420.790776:0:12100:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:3.0:1773941420.790781:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5ba0c90e00. 00000001:00000010:3.0:1773941420.790785:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5ba0c91a00. 00000001:00000010:3.0:1773941420.790788:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5ba0c90c00. 00000001:00000010:3.0:1773941420.790791:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5ba0c91200. 00000001:00000010:3.0:1773941420.790795:0:12100:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 32 at ffff8b5b88570bc0. 00000001:00000010:3.0:1773941420.790798:0:12100:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 232 at ffff8b5ba0c22a00. 00000001:00000001:3.0:1773941420.790805:0:12100:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00000020:00000040:3.0:1773941420.790806:0:12100:0:(obd_config.c:1058:class_decref()) Decref lustre-OST0000-osc-ffff8b5b885e5800 (ffff8b5ba07399b8) now 3 - setup 00000020:00000001:3.0:1773941420.790808:0:12100:0:(obd_config.c:1021:class_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.790810:0:12100:0:(obd_config.c:1659:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000080:3.0:1773941420.790812:0:12100:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000001:3.0:1773941420.790815:0:12100:0:(obd_config.c:899:class_detach()) Process entered 00000020:00000040:3.0:1773941420.790817:0:12100:0:(obd_config.c:1058:class_decref()) Decref lustre-OST0000-osc-ffff8b5b885e5800 (ffff8b5ba07399b8) now 2 - obd_device_list 00000020:00000080:3.0:1773941420.790819:0:12100:0:(obd_config.c:919:class_detach()) detach on obd lustre-OST0000-osc-ffff8b5b885e5800 (uuid 637f784d-29a5-48a3-b831-1c4c7c7a2a54) 00000020:00000040:3.0:1773941420.790820:0:12100:0:(obd_config.c:1058:class_decref()) Decref lustre-OST0000-osc-ffff8b5b885e5800 (ffff8b5ba07399b8) now 1 - newdev 00000020:00000040:3.0:1773941420.790823:0:12100:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b88956000 with handle 0xa33a5be14eae1c05 from hash 00000020:00000040:3.0:1773941420.790825:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b88956000 : new refcount 0 00000020:00000080:3.0:1773941420.790826:0:12100:0:(genops.c:904:class_export_put()) final put ffff8b5b88956000/637f784d-29a5-48a3-b831-1c4c7c7a2a54 00000020:00000001:3.0:1773941420.790828:0:12100:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.790829:0:12100:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:3.0:1773941420.790830:0:12100:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b88956000/637f784d-29a5-48a3-b831-1c4c7c7a2a54 for lustre-OST0000-osc-ffff8b5b885e5800 00000020:00000001:3.0:1773941420.790832:0:12100:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:3.0:1773941420.790833:0:12100:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.790835:0:12100:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff8b5b88956000. 00000020:00000001:3.0:1773941420.790838:0:12100:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000040:3.0:1773941420.790840:0:12100:0:(genops.c:437:class_free_dev()) Release obd device lustre-OST0000-osc-ffff8b5b885e5800 obd_type name = osc 00000020:01000000:3.0:1773941420.790841:0:12100:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-OST0000-osc-ffff8b5b885e5800 (637f784d-29a5-48a3-b831-1c4c7c7a2a54) 00000020:00000001:3.0:1773941420.790843:0:12100:0:(obd_class.h:633:obd_cleanup()) Process entered 00000020:00000010:3.0:1773941420.790848:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84e8e400. 00000020:00000010:3.0:1773941420.790850:0:12100:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b885b0600. 00000020:00000010:3.0:1773941420.790852:0:12100:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc91e0. 00000008:00000010:3.0:1773941420.790854:0:12100:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8b5ba0f0d940. 00020000:00000010:3.0:1773941420.790856:0:12100:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8b5b89382c00. 00008000:00000010:3.0:1773941420.790858:0:12100:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b98a1e6d8. 00000008:00000010:3.0:1773941420.790863:0:12100:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5ba0f02ca0. 00000008:00000010:3.0:1773941420.790867:0:12100:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5ba0f0e468. 00000008:00000010:3.0:1773941420.790870:0:12100:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5b90d8c2f0. 00000008:00000010:3.0:1773941420.790872:0:12100:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5b90d8a178. 00000008:00000010:3.0:1773941420.790874:0:12100:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5b90d8ce18. 00000008:00000010:3.0:1773941420.790877:0:12100:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5ba0f05940. 00000008:00000010:3.0:1773941420.790881:0:12100:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5ba0f06468. 00000008:00000010:3.0:1773941420.790884:0:12100:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5ba0f037c8. 00000008:00000010:3.0:1773941420.790887:0:12100:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5ba0f06f90. 00000008:00000010:3.0:1773941420.790890:0:12100:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8b5b885b29a0. 00000008:00000010:3.0:1773941420.790892:0:12100:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8b5ba0c338c0. 00000008:00000010:3.0:1773941420.790895:0:12100:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8b5ba0b98b00. 00000008:00000010:3.0:1773941420.790898:0:12100:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8b5ba0b98580. 00000008:00000010:3.0:1773941420.790900:0:12100:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8b5ba0b99340. 00000008:00000010:3.0:1773941420.790902:0:12100:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8b5b885b2dc0. 00000008:00000010:3.0:1773941420.790916:0:12100:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8b5b885b2840. 00000008:00000010:3.0:1773941420.790918:0:12100:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8b5b885b3760. 00000008:00000010:3.0:1773941420.790920:0:12100:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8b5b885b3b80. 00000008:00000001:3.0:1773941420.790921:0:12100:0:(osc_request.c:4146:osc_cleanup_common()) Process entered 00010000:00000001:3.0:1773941420.790925:0:12100:0:(ldlm_lib.c:631:client_obd_cleanup()) Process entered 00010000:00000001:3.0:1773941420.790927:0:12100:0:(ldlm_resource.c:1304:ldlm_namespace_free_post()) Process entered 00010000:00000001:3.0:1773941420.790929:0:12100:0:(ldlm_pool.c:931:ldlm_pool_fini()) Process entered 00000020:00000010:3.0:1773941420.790947:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 352 at ffff8b5b850b7200. 00000020:00000010:3.0:1773941420.790949:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5ba0734000. 00010000:00000001:3.0:1773941420.790970:0:12100:0:(ldlm_pool.c:934:ldlm_pool_fini()) Process leaving 00000020:00000010:3.0:1773941420.790979:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 96 at ffff8b5b880f9960. 00000020:00000010:3.0:1773941420.790981:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5ba0737400. 00000001:00000001:3.0:1773941420.791020:0:12100:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:3.0:1773941420.791054:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5b850bc000. 00000001:00000010:3.0:1773941420.791062:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5b850be000. 00000001:00000010:3.0:1773941420.791064:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5b850ba000. 00000001:00000010:3.0:1773941420.791067:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5b850b8000. 00000001:00000010:3.0:1773941420.791069:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5b885c6000. 00000001:00000010:3.0:1773941420.791072:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5b885c2000. 00000001:00000010:3.0:1773941420.791074:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5b885c0000. 00000001:00000010:3.0:1773941420.791076:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5b885c4000. 00000001:00000010:3.0:1773941420.791079:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5b885ca000. 00000001:00000010:3.0:1773941420.791082:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5b885c8000. 00000001:00000010:3.0:1773941420.791086:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5b885cc000. 00000001:00000010:3.0:1773941420.791089:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5b885ce000. 00000001:00000010:3.0:1773941420.791093:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5ba0cb6000. 00000001:00000010:3.0:1773941420.791097:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5ba0cb2000. 00000001:00000010:3.0:1773941420.791101:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5ba0cb0000. 00000001:00000010:3.0:1773941420.791104:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff8b5ba0cb4000. 00000001:00000010:3.0:1773941420.791108:0:12100:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 128 at ffff8b5b90edbb80. 00000001:00000010:3.0:1773941420.791112:0:12100:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 280 at ffff8b5b850b7600. 00000001:00000001:3.0:1773941420.791115:0:12100:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00010000:00000010:3.0:1773941420.791118:0:12100:0:(ldlm_resource.c:1327:ldlm_namespace_free_post()) kfreed 'ns': 848 at ffff8b5ba0736000. 00010000:00000001:3.0:1773941420.791122:0:12100:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:3.0:1773941420.791124:0:12100:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00010000:00000001:3.0:1773941420.791126:0:12100:0:(ldlm_resource.c:1329:ldlm_namespace_free_post()) Process leaving 00010000:00000001:3.0:1773941420.791127:0:12100:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 00010000:00000001:3.0:1773941420.791130:0:12100:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 00010000:00000001:3.0:1773941420.791131:0:12100:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:3.0:1773941420.791133:0:12100:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00010000:00000010:3.0:1773941420.791135:0:12100:0:(ldlm_lib.c:647:client_obd_cleanup()) kfreed 'cli->cl_read_io_latency_by_size': 3920 at ffff8b5b90f36000. 00010000:00000010:3.0:1773941420.791140:0:12100:0:(ldlm_lib.c:651:client_obd_cleanup()) kfreed 'cli->cl_write_io_latency_by_size': 3920 at ffff8b5b90f33000. 00010000:00000001:3.0:1773941420.791143:0:12100:0:(ldlm_lib.c:654:client_obd_cleanup()) Process leaving 00000008:00000001:3.0:1773941420.791144:0:12100:0:(osc_request.c:4167:osc_cleanup_common()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:3.0:1773941420.791146:0:12100:0:(osc_dev.c:179:osc_device_free()) kfreed 'oc': 104 at ffff8b5b90edb300. 00000020:00000010:3.0:1773941420.791149:0:12100:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b885b0600. 00000020:00000010:3.0:1773941420.791152:0:12100:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc91e0. 00000008:00000010:3.0:1773941420.791154:0:12100:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5ba0f0d940. 00020000:00000010:3.0:1773941420.791161:0:12100:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b89382c00. 00008000:00000010:3.0:1773941420.791164:0:12100:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b98a1e6d8. 00000020:00000010:3.0:1773941420.791165:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84e8e400. 00000020:00000001:3.0:1773941420.791168:0:12100:0:(obd_class.h:647:obd_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.791170:0:12100:0:(genops.c:66:obd_device_free()) slab-freed '(obd)': 6584 at ffff8b5ba07399b8. 00000020:00000001:3.0:1773941420.791175:0:12100:0:(obd_config.c:923:class_detach()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.791177:0:12100:0:(obd_config.c:1655:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1773941420.791179:0:12100:0:(obd_config.c:2523:class_manual_cleanup()) kfreed 'lcfg': 88 at ffff8b5ba0bc9a20. 00000020:00000001:3.0:1773941420.791181:0:12100:0:(obd_config.c:2524:class_manual_cleanup()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1773941420.791183:0:12100:0:(lu_tgt_descs.c:285:lu_tgt_descs_fini()) kfreed 'ltd->ltd_tgt_idx[i]': 4096 at ffff8b5b90f34000. 00020000:00000001:3.0:1773941420.791186:0:12100:0:(lov_obd.c:795:lov_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.791195:0:12100:0:(lu_object.c:1212:lu_site_fini()) kvfree 's->ls_bkts': 18432 at ffff8b5b90dd0000. 00000020:00000010:3.0:1773941420.791203:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 192 at ffff8b5b885de000. 00000020:00000010:3.0:1773941420.791205:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5ba0ce6c00. 00020000:00000010:3.0:1773941420.791209:0:12100:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a2fd180. 00020000:00000010:3.0:1773941420.791211:0:12100:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b893820c0. 00020000:00000010:3.0:1773941420.791213:0:12100:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a10dcc0. 00020000:00000010:3.0:1773941420.791215:0:12100:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a10d840. 00020000:00000010:3.0:1773941420.791218:0:12100:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a10df00. 00020000:00000010:3.0:1773941420.791220:0:12100:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a2fd240. 00020000:00000010:3.0:1773941420.791222:0:12100:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a2fdd80. 00020000:00000010:3.0:1773941420.791224:0:12100:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a2fd600. 00020000:00000010:3.0:1773941420.791226:0:12100:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a2fd3c0. 00020000:00000010:3.0:1773941420.791229:0:12100:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5b84cbfa80. 00020000:00000010:3.0:1773941420.791231:0:12100:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5ba0c34d80. 00020000:00000010:3.0:1773941420.791233:0:12100:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5ba0b9e1c0. 00020000:00000010:3.0:1773941420.791236:0:12100:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5ba0b9fa80. 00020000:00000010:3.0:1773941420.791239:0:12100:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5ba0b9f840. 00020000:00000010:3.0:1773941420.791240:0:12100:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5b84cbfcc0. 00020000:00000010:3.0:1773941420.791242:0:12100:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5b84cbd8c0. 00020000:00000010:3.0:1773941420.791243:0:12100:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5b84cbe640. 00020000:00000010:3.0:1773941420.791245:0:12100:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5b84cbc480. 00020000:00000010:3.0:1773941420.791247:0:12100:0:(lov_dev.c:306:lov_device_free()) kfreed 'ld->ld_target': 512 at ffff8b5b850b7e00. 00020000:00000010:3.0:1773941420.791253:0:12100:0:(lov_dev.c:310:lov_device_free()) kfreed 'ld->ld_md_tgts': 4096 at ffff8b5b85e5f000. 00020000:00000010:3.0:1773941420.791256:0:12100:0:(lov_dev.c:315:lov_device_free()) kfreed 'ld->ld_lov->lov_mdc_tgts': 4096 at ffff8b5b85e5a000. 00020000:00000010:3.0:1773941420.791258:0:12100:0:(lov_dev.c:319:lov_device_free()) kfreed 'ld': 632 at ffff8b5ba0ce6000. 00000020:00001000:3.0:1773941420.791261:0:12100:0:(cl_object.c:925:cl_env_put()) 1@ffff8b5b8509c2d0 00000020:00000010:3.0:1773941420.791263:0:12100:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b89316400. 00000020:00000010:3.0:1773941420.791265:0:12100:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0166a20. 00000008:00000010:3.0:1773941420.791280:0:12100:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8b5ba0f04e18. 00020000:00000010:3.0:1773941420.791283:0:12100:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8b5b8a2fd780. 00008000:00000010:3.0:1773941420.791286:0:12100:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d77880. 00000080:00000010:3.0:1773941420.791289:0:12100:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8b5b85eeca80. 00000080:00000010:3.0:1773941420.791292:0:12100:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8b5b98a8b908. 00000020:00000010:3.0:1773941420.791294:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b89316600. 00000008:00000010:3.0:1773941420.791297:0:12100:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8b5b885b3e40. 00020000:00000010:3.0:1773941420.791300:0:12100:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8b5b84cbc240. 00008000:00000010:3.0:1773941420.791303:0:12100:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8b5b82c37248. 00000080:00000010:3.0:1773941420.791306:0:12100:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8b5b845f5780. 00000020:00000010:3.0:1773941420.791308:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b89317000. 00000020:00000010:3.0:1773941420.791309:0:12100:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8b5b8509c2d0. 00000080:00000001:3.0:1773941420.791310:0:12100:0:(vvp_dev.c:331:cl_sb_fini()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1773941420.791313:0:12100:0:(obd_class.h:902:obd_disconnect()) Process entered 00020000:00000001:3.0:1773941420.791315:0:12100:0:(lov_obd.c:290:lov_disconnect()) Process entered 00000020:00000001:3.0:1773941420.791315:0:12100:0:(genops.c:1462:class_disconnect()) Process entered 00000020:00000080:3.0:1773941420.791317:0:12100:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0xa33a5be14eae1c28 00000020:00000040:3.0:1773941420.791319:0:12100:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b90f27000 with handle 0xa33a5be14eae1c28 from hash 00000020:00000001:3.0:1773941420.791322:0:12100:0:(genops.c:1934:obd_stale_export_put()) Process entered 00000020:00000040:3.0:1773941420.791323:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b90f27000 : new refcount 1 00000020:00000001:3.0:1773941420.791324:0:12100:0:(genops.c:1959:obd_stale_export_put()) Process leaving 00000020:00000040:3.0:1773941420.791326:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b90f27000 : new refcount 0 00000020:00000080:3.0:1773941420.791327:0:12100:0:(genops.c:904:class_export_put()) final put ffff8b5b90f27000/637f784d-29a5-48a3-b831-1c4c7c7a2a54 00000020:00000001:3.0:1773941420.791329:0:12100:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.791337:0:12100:0:(genops.c:1493:class_disconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1773941420.791339:0:12100:0:(lov_obd.c:321:lov_disconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1773941420.791340:0:12100:0:(obd_class.h:914:obd_disconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941420.791360:0:37:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:0.0:1773941420.791364:0:37:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b90f27000/637f784d-29a5-48a3-b831-1c4c7c7a2a54 for lustre-clilov-ffff8b5b885e5800 00000020:00000001:0.0:1773941420.791367:0:37:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:0.0:1773941420.791369:0:37:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941420.791372:0:37:0:(obd_config.c:1058:class_decref()) Decref lustre-clilov-ffff8b5b885e5800 (ffff8b5b85513370) now 4 - export 00000020:00000010:0.0:1773941420.791375:0:37:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff8b5b90f27000. 00000020:00000001:0.0:1773941420.791382:0:37:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000010:3.0:1773941420.791549:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 576 at ffff8b5b90263c00. 00000020:00000010:3.0:1773941420.791559:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5b90262c00. 00000020:00000010:3.0:1773941420.791562:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 1856 at ffff8b5b88968000. 00000020:00000010:3.0:1773941420.791565:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 1856 at ffff8b5b90f22800. 00000020:00000010:3.0:1773941420.791570:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 1440 at ffff8b5b98ac3000. 00000020:00000010:3.0:1773941420.791575:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5b90261c00. 00000080:00000001:3.0:1773941420.791578:0:12100:0:(obd_class.h:902:obd_disconnect()) Process entered 00800000:00000001:3.0:1773941420.791581:0:12100:0:(lmv_obd.c:589:lmv_disconnect()) Process entered 00800000:00000001:3.0:1773941420.791584:0:12100:0:(lmv_obd.c:534:lmv_disconnect_mdc()) Process entered 00020000:00000001:3.0:1773941420.791588:0:12100:0:(lu_tgt_descs.c:164:lu_qos_del_tgt()) Process entered 00020000:00001000:3.0:1773941420.791589:0:12100:0:(lu_tgt_descs.c:175:lu_qos_del_tgt()) removing server 192.168.204.143@tcp 00020000:00000010:3.0:1773941420.791591:0:12100:0:(lu_tgt_descs.c:177:lu_qos_del_tgt()) kfreed 'svr': 104 at ffff8b5b90edbe00. 00020000:00000001:3.0:1773941420.791593:0:12100:0:(lu_tgt_descs.c:186:lu_qos_del_tgt()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:3.0:1773941420.791595:0:12100:0:(fld_request.c:185:fld_client_del_target()) Process entered 00000020:00000040:3.0:1773941420.791596:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b88956800 : new refcount 3 80000000:00000010:3.0:1773941420.791598:0:12100:0:(fld_request.c:197:fld_client_del_target()) kfreed 'target': 40 at ffff8b5b850b0340. 80000000:00000001:3.0:1773941420.791600:0:12100:0:(fld_request.c:198:fld_client_del_target()) Process leaving (rc=0 : 0 : 0) 40000000:00000001:3.0:1773941420.791602:0:12100:0:(fid_request.c:460:client_fid_fini()) Process entered 40000000:00000001:3.0:1773941420.791603:0:12100:0:(fid_request.c:376:seq_client_fini()) Process entered 00000020:00000040:3.0:1773941420.791653:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b88956800 : new refcount 2 40000000:00000001:3.0:1773941420.791656:0:12100:0:(fid_request.c:386:seq_client_fini()) Process leaving 40000000:00000010:3.0:1773941420.791658:0:12100:0:(fid_request.c:465:client_fid_fini()) kfreed 'cli->cl_seq': 224 at ffff8b5b9024be00. 40000000:00000001:3.0:1773941420.791661:0:12100:0:(fid_request.c:470:client_fid_fini()) Process leaving (rc=0 : 0 : 0) 00800000:00000040:3.0:1773941420.791663:0:12100:0:(lmv_obd.c:568:lmv_disconnect_mdc()) Disconnected from lustre-MDT0000-mdc-ffff8b5b885e5800(637f784d-29a5-48a3-b831-1c4c7c7a2a54) successfully 00800000:00000001:3.0:1773941420.791665:0:12100:0:(obd_class.h:1347:obd_register_observer()) Process entered 00800000:00000001:3.0:1773941420.791667:0:12100:0:(obd_class.h:1360:obd_register_observer()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:3.0:1773941420.791670:0:12100:0:(obd_class.h:902:obd_disconnect()) Process entered 00010000:00000001:3.0:1773941420.791673:0:12100:0:(ldlm_lib.c:759:client_disconnect_export()) Process entered 00010000:00000040:3.0:1773941420.791675:0:12100:0:(ldlm_lib.c:772:client_disconnect_export()) disconnect lustre-MDT0000-mdc-ffff8b5b885e5800 - 1 00000100:00000001:3.0:1773941420.791678:0:12100:0:(pinger.c:388:ptlrpc_pinger_del_import()) Process entered 00000100:00080000:3.0:1773941420.791681:0:12100:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 637f784d-29a5-48a3-b831-1c4c7c7a2a54->lustre-MDT0000_UUID 00000020:00000001:3.0:1773941420.791682:0:12100:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941420.791683:0:12100:0:(genops.c:1242:class_import_put()) import ffff8b5b88953800 refcount=2 obd=lustre-MDT0000-mdc-ffff8b5b885e5800 00000020:00000001:3.0:1773941420.791684:0:12100:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941420.791685:0:12100:0:(pinger.c:401:ptlrpc_pinger_del_import()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.791687:0:12100:0:(ldlm_request.c:2367:ldlm_cli_cancel_unused()) Process entered 00000001:00000001:3.0:1773941420.791693:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.791694:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00010000:00000040:3.0:1773941420.791798:0:12100:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8b5b890ec400 count: 2 00010000:00000001:3.0:1773941420.791802:0:12100:0:(ldlm_request.c:2311:ldlm_cli_cancel_unused_resource()) Process entered 00010000:00000040:3.0:1773941420.791804:0:12100:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8b5b890ec400 count: 3 00010000:00000001:3.0:1773941420.791806:0:12100:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1773941420.791812:0:12100:0:(ldlm_request.c:2161:ldlm_cancel_resource_local()) Process entered 00010000:00000001:3.0:1773941420.791815:0:12100:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:3.0:1773941420.791818:0:12100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00000001:3.0:1773941420.791825:0:12100:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:3.0:1773941420.791828:0:12100:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:3.0:1773941420.791830:0:12100:0:(namei.c:251:ll_lock_cancel_bits()) Process leaving 00000080:00000001:3.0:1773941420.791831:0:12100:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.791833:0:12100:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:3.0:1773941420.791835:0:12100:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941420.791836:0:12100:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941420.791838:0:12100:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941420.791840:0:12100:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1773941420.791842:0:12100:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138ea00 with handle 0xa33a5be14eae1c36 from hash 00010000:00000001:3.0:1773941420.791843:0:12100:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941420.791845:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941420.791846:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941420.791847:0:12100:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000020:00000010:3.0:1773941420.791850:0:12100:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 448 at ffff8b5b885b0600. 00010000:00000001:3.0:1773941420.791853:0:12100:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941420.791854:0:12100:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:3.0:1773941420.791857:0:12100:0:(ldlm_request.c:1740:ldlm_cli_cancel_list_local()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1773941420.791858:0:12100:0:(ldlm_request.c:2211:ldlm_cancel_resource_local()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1773941420.791860:0:12100:0:(ldlm_request.c:2242:ldlm_cli_cancel_list()) Process entered 00010000:00000001:3.0:1773941420.791861:0:12100:0:(ldlm_request.c:1396:ldlm_cli_cancel_req()) Process entered 00000100:00000010:3.0:1773941420.791866:0:12100:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275f80. 00000020:00000040:3.0:1773941420.791868:0:12100:0:(genops.c:1229:class_import_get()) import ffff8b5b88953800 refcount=3 obd=lustre-MDT0000-mdc-ffff8b5b885e5800 00000100:00000001:3.0:1773941420.791871:0:12100:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941420.791876:0:12100:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941420.791878:0:12100:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941420.791882:0:12100:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b885b0400. 00000100:00000001:3.0:1773941420.791886:0:12100:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.791888:0:12100:0:(ldlm_request.c:1348:ldlm_cancel_pack()) Process entered 00010000:00010000:3.0:1773941420.791893:0:12100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:3.0:1773941420.791898:0:12100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:3.0:1773941420.791899:0:12100:0:(ldlm_request.c:1380:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941420.791900:0:12100:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941420.791901:0:12100:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941420.791905:0:12100:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b90ec5d80. 00000100:00000001:3.0:1773941420.791906:0:12100:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824122535296 : -128249587016320 : ffff8b5b90ec5d80) 00000100:00000001:3.0:1773941420.791908:0:12100:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941420.791909:0:12100:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109625216 : -128249599926400 : ffff8b5b90275f80) 00000100:00000001:3.0:1773941420.791912:0:12100:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941420.791913:0:12100:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.791915:0:12100:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941420.791916:0:12100:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941420.791919:0:12100:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275f80 x1860112241143936/t0(0) o103->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.791924:0:12100:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941420.791925:0:12100:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941420.791927:0:12100:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275f80 x1860112241143936/t0(0) o103->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.791932:0:12100:0:(client.c:1841:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:3.0:1773941420.791934:0:12100:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941420.791934:0:12100:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941420.791936:0:12100:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b90ec5d80 going to sleep for 0 seconds 00000100:00000001:3.0:1773941420.791937:0:12100:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941420.791938:0:12100:0:(client.c:2089:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:3.0:1773941420.791940:0:12100:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941420.791945:0:12100:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941420.791946:0:12100:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941420.791949:0:12100:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275f80 x1860112241143936/t0(0) o103->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941420.791954:0:12100:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275f80 pname:cluuid:pid:xid:nid:opc:job umount:637f784d-29a5-48a3-b831-1c4c7c7a2a54:12100:1860112241143936:192.168.204.143@tcp:103:umount.0 00000100:00000001:3.0:1773941420.791956:0:12100:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941420.791958:0:12100:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:3.0:1773941420.791959:0:12100:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941420.791960:0:12100:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941420.791962:0:12100:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275f80 x1860112241143936/t0(0) o103->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.791966:0:12100:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941420.791967:0:12100:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b90ec5d80. 00000100:00000001:3.0:1773941420.791969:0:12100:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941420.791970:0:12100:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00010000:3.0:1773941420.791972:0:12100:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00000100:00000001:3.0:1773941420.791973:0:12100:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941420.791974:0:12100:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275f80 x1860112241143936/t0(0) o103->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.791978:0:12100:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1773941420.791979:0:12100:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941420.791979:0:12100:0:(genops.c:1242:class_import_put()) import ffff8b5b88953800 refcount=2 obd=lustre-MDT0000-mdc-ffff8b5b885e5800 00000020:00000001:3.0:1773941420.791981:0:12100:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941420.791982:0:12100:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b885b0400. 02000000:00000001:3.0:1773941420.791984:0:12100:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941420.791984:0:12100:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941420.791985:0:12100:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275f80. 00000100:00000001:3.0:1773941420.791987:0:12100:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941420.791989:0:12100:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1773941420.791990:0:12100:0:(ldlm_request.c:1501:ldlm_cli_cancel_req()) Process leaving 00010000:00010000:3.0:1773941420.791991:0:12100:0:(ldlm_request.c:2277:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00000001:3.0:1773941420.791992:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941420.791994:0:12100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b5b885e5800 lock: ffff8b5b8138ea00/0xa33a5be14eae1c36 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xb574482a8fd85fac expref: -99 pid: 12084 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000020:00000010:3.0:1773941420.792004:0:12100:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff8b5b9020a200. 00010000:00000001:3.0:1773941420.792009:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941420.792010:0:12100:0:(ldlm_request.c:2290:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.792011:0:12100:0:(ldlm_request.c:2328:ldlm_cli_cancel_unused_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1773941420.792013:0:12100:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec400 count: 0 00010000:00000010:3.0:1773941420.792017:0:12100:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8b5b90edb600. 00000001:00000001:3.0:1773941420.792263:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.792279:0:12100:0:(ldlm_request.c:2379:ldlm_cli_cancel_unused()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.792282:0:12100:0:(ldlm_resource.c:1268:ldlm_namespace_free_prior()) Process entered 00010000:00000001:3.0:1773941420.792283:0:12100:0:(ldlm_resource.c:1207:__ldlm_namespace_free()) Process entered 00000001:00000001:3.0:1773941420.792284:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.792285:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.792571:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.792573:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.792574:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.792888:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.792890:0:12100:0:(ldlm_resource.c:1251:__ldlm_namespace_free()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.792891:0:12100:0:(ldlm_resource.c:1293:ldlm_namespace_free_prior()) Process leaving 00000100:00000001:3.0:1773941420.792892:0:12100:0:(import.c:1924:ptlrpc_disconnect_import()) Process entered 00000100:00000001:3.0:1773941420.792894:0:12100:0:(import.c:1858:ptlrpc_disconnect_import_async()) Process entered 00000100:00000001:3.0:1773941420.792896:0:12100:0:(import.c:1750:ptlrpc_disconnect_prep_req()) Process entered 00000100:00000010:3.0:1773941420.792898:0:12100:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275f80. 00000020:00000040:3.0:1773941420.792900:0:12100:0:(genops.c:1229:class_import_get()) import ffff8b5b88953800 refcount=3 obd=lustre-MDT0000-mdc-ffff8b5b885e5800 00000100:00000001:3.0:1773941420.792902:0:12100:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941420.792903:0:12100:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941420.792905:0:12100:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941420.792907:0:12100:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b98a69500. 00000100:00000001:3.0:1773941420.792910:0:12100:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.792911:0:12100:0:(import.c:1788:ptlrpc_disconnect_prep_req()) Process leaving (rc=18446615824109625216 : -128249599926400 : ffff8b5b90275f80) 00000100:00080000:3.0:1773941420.792913:0:12100:0:(import.c:65:import_set_state_nolock()) ffff8b5b88953800 lustre-MDT0000_UUID: changing import state from FULL to CONNECTING 00000100:00000001:3.0:1773941420.792915:0:12100:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941420.792916:0:12100:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941420.792923:0:12100:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a88f5799] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90275f80 x1860112241144064/t0(0) o39->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.792937:0:12100:0:(import.c:1904:ptlrpc_disconnect_import_async()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.792958:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941420.792961:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941420.792962:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941420.792967:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275f80 x1860112241144064/t0(0) o39->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941420.792974:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941420.792976:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941420.792979:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941420.792981:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941420.792984:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941420.792987:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:637f784d-29a5-48a3-b831-1c4c7c7a2a54:2351:1860112241144064:192.168.204.143@tcp:39:umount.0 00000100:00000001:2.0:1773941420.792992:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941420.792996:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941420.792998:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941420.793000:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941420.793003:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b89317400. 02000000:00000001:2.0:1773941420.793005:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941420.793008:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941420.793011:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af97e8. 00000400:00000010:2.0:1773941420.793015:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f344c8. 00000100:00000200:2.0:1773941420.793018:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241144064, portal 10 00000100:00000001:2.0:1773941420.793019:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941420.793020:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109625216 : -128249599926400 : ffff8b5b90275f80) 00000100:00000040:2.0:1773941420.793024:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275f80 x1860112241144064/t0(0) o39->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941420.793029:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941420.793030:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941420.793033:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34880. 00000100:00000200:2.0:1773941420.793035:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1860112241144064, offset 0 00000400:00000200:2.0:1773941420.793039:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941420.793051:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941420.793060:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941420.793071:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941420.793075:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941420.793078:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941420.793081:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941420.793085:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [104:104:256:4294967295] 192.168.204.143@tcp LPNI seq info [104:104:8:4294967295] 00000400:00000200:2.0:1773941420.793090:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941420.793098:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941420.793101:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941420.793105:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0b00. 00000800:00000200:2.0:1773941420.793110:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941420.793114:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941420.793117:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0b00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1773941420.793127:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941420.793131:0:2351:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b5b885e5800: using a timeout of 1429561 usecs (1430 jiffies) 00000400:00000200:2.0:1773941420.793135:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941420.793138:0:2351:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b5b885e5800: PM QoS new deadline estimation for cpu 2 is 4294860015 00000100:00000001:2.0:1773941420.793141:0:2351:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b5b885e5800: PM QoS request active for cpu 2, keep current deadline 4294861147 00000100:00000001:2.0:1773941420.793143:0:2351:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b5b885e5800: PM QoS new deadline estimation for cpu 3 is 4294860015 00000100:00000001:2.0:1773941420.793145:0:2351:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b5b885e5800: PM QoS request active for cpu 3, keep current deadline 4294861147 00000100:00000001:2.0:1773941420.793147:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941420.793149:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.793152:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.793154:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941420.793158:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941420.793159:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941420.793162:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941420.793163:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941420.793165:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.793168:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941420.793207:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941420.793211:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0b00. 00000400:00000200:3.0:1773941420.793214:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941420.793217:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941420.793220:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34880 00000400:00000010:3.0:1773941420.793222:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34880. 00000100:00000001:3.0:1773941420.793224:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941420.793226:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275f80 x1860112241144064/t0(0) o39->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.793231:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941420.793233:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275f80 x1860112241144064/t0(0) o39->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941430 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.793237:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.793238:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941420.795158:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.795165:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941420.795167:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.795170:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.795174:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.795178:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941420.795183:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.795186:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.795189:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.795191:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 10 MB=0x69bc321603100 00000400:00000200:3.0:1773941420.795194:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.795197:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.143@tcp of length 224/224 into md 0x895 [1] + 224 00000800:00000001:3.0:1773941420.795202:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.795206:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.795208:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941420.795216:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941420.795219:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941420.795223:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275f80 x1860112241144064/t0(0) o39->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941420.795230:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b90275f80 x1860112241144064/t0(0) o39->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.795243:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941420.795248:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.795250:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941420.795279:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941420.795282:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941420.795287:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941420.795289:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941420.795292:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af97e8. 00000400:00000200:1.0:1773941420.795296:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f344c8 00000400:00000010:1.0:1773941420.795298:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f344c8. 00000100:00000001:1.0:1773941420.795301:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941420.795306:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275f80 x1860112241144064/t0(0) o39->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941420.795314:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275f80 x1860112241144064/t0(0) o39->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941420.795320:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941420.795322:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941420.795324:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941420.795326:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941420.795328:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941420.795330:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941420.795332:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941420.795337:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b88953c58 time=1 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941420.795341:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b88953c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941420.795343:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941420.795345:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941420.795347:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941420.795350:0:2351:0:(ldlm_request.c:1546:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.795353:0:2351:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00100000:1.0:1773941420.795355:0:2351:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b5b885e5800: committing for last_committed 8589934593 gen 1 00000100:00000040:1.0:1773941420.795358:0:2351:0:(client.c:3191:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b5b885e5800: normal: skip=0 replay=0/0 committed=0/0 00000100:00000001:1.0:1773941420.795360:0:2351:0:(client.c:3192:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1773941420.795362:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941420.795367:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275f80 x1860112241144064/t8589934593(8589934593) o39->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941420.795374:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941420.795382:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941420.795383:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941420.795386:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b88953800 lustre-MDT0000_UUID: changing import state from CONNECTING to CLOSED 00000100:00000001:1.0:1773941420.795388:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:1.0:1773941420.795390:0:2351:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000002:00000001:1.0:1773941420.795393:0:2351:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.795395:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941420.795396:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:1.0:1773941420.795397:0:2351:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000002:00000001:1.0:1773941420.795399:0:2351:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.795400:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000040:1.0:1773941420.795413:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275f80 x1860112241144064/t8589934593(8589934593) o39->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941420.795421:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:637f784d-29a5-48a3-b831-1c4c7c7a2a54:2351:1860112241144064:192.168.204.143@tcp:39:umount.0 00000100:00000001:1.0:1773941420.795425:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941420.795427:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941420.795430:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275f80 x1860112241144064/t8589934593(8589934593) o39->lustre-MDT0000-mdc-ffff8b5b885e5800@192.168.204.143@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.795431:0:12100:0:(import.c:1967:ptlrpc_disconnect_import()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.795436:0:12100:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941420.795439:0:12100:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0000_UUID INVALID 00000100:00000001:3.0:1773941420.795441:0:12100:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:3.0:1773941420.795443:0:12100:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00000001:1.0:1773941420.795444:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:3.0:1773941420.795445:0:12100:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b5b885e5800: committing for last_committed 8589934593 gen 2 00000100:00000040:3.0:1773941420.795448:0:12100:0:(client.c:3191:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b5b885e5800: normal: skip=0 replay=0/0 committed=0/0 00000100:00000001:3.0:1773941420.795450:0:12100:0:(client.c:3192:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1773941420.795450:0:12100:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941420.795451:0:12100:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941420.795452:0:12100:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:3.0:1773941420.795454:0:12100:0:(mdc_request.c:2862:mdc_import_event()) Process entered 02000000:00000001:1.0:1773941420.795454:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000002:00000001:3.0:1773941420.795458:0:12100:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.795459:0:12100:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:1.0:1773941420.795459:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b89317400. 00000100:00100000:3.0:1773941420.795462:0:12100:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941420.795464:0:12100:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:1.0:1773941420.795464:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000002:00000001:3.0:1773941420.795465:0:12100:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000001:00000001:3.0:1773941420.795467:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:1.0:1773941420.795468:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:3.0:1773941420.795469:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:1.0:1773941420.795471:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b88953800 refcount=2 obd=lustre-MDT0000-mdc-ffff8b5b885e5800 00000020:00000001:1.0:1773941420.795474:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941420.795476:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b98a69500. 02000000:00000001:1.0:1773941420.795478:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941420.795480:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941420.795482:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275f80. 00000100:00000001:1.0:1773941420.795484:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941420.795485:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941420.795487:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941420.795492:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941420.795493:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941420.795502:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941420.795503:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.795794:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.795796:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.795797:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.796064:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.796066:0:12100:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1773941420.796069:0:12100:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8b5b850f85a0. 00000020:00000010:3.0:1773941420.796072:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b885b0400. 00000020:00000010:3.0:1773941420.796079:0:12100:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b885b1600. 00000020:00000010:3.0:1773941420.796082:0:12100:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9a20. 00008000:00000010:3.0:1773941420.796086:0:12100:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b98a1e6d8. 00000020:00000010:3.0:1773941420.796089:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84e8e400. 00008000:00000010:3.0:1773941420.796091:0:12100:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5b98a6ba30. 00000020:00000001:3.0:1773941420.796093:0:12100:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446615823923512744 : -128249786038872 : ffff8b5b850f85a8) 00000020:00001000:3.0:1773941420.796095:0:12100:0:(cl_object.c:831:cl_env_get()) 1@ffff8b5b850f85a0 00000008:00000001:3.0:1773941420.796097:0:12100:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1773941420.796098:0:12100:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1773941420.796100:0:12100:0:(osc_cache.c:2316:osc_next_obj()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1773941420.796110:0:12100:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000001:00000001:3.0:1773941420.796111:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.796112:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.796393:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:3.0:1773941420.796395:0:12100:0:(cl_object.c:925:cl_env_put()) 1@ffff8b5b850f85a0 00000020:00000010:3.0:1773941420.796398:0:12100:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b885b1600. 00000020:00000010:3.0:1773941420.796401:0:12100:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9a20. 00008000:00000010:3.0:1773941420.796403:0:12100:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b98a1e6d8. 00000020:00000010:3.0:1773941420.796405:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b885b0400. 00008000:00000010:3.0:1773941420.796407:0:12100:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8b5b98a6ba30. 00000020:00000010:3.0:1773941420.796409:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84e8e400. 00000020:00000010:3.0:1773941420.796410:0:12100:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8b5b850f85a0. 00000001:00000001:3.0:1773941420.796412:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.796413:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.796840:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.796842:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.796843:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.797111:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:3.0:1773941420.797112:0:12100:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.797114:0:12100:0:(obd_class.h:1226:obd_import_event()) Process leaving 00010000:00000001:3.0:1773941420.797117:0:12100:0:(ldlm_lib.c:818:client_disconnect_export()) Process leaving 00000020:00000001:3.0:1773941420.797118:0:12100:0:(genops.c:1462:class_disconnect()) Process entered 00000020:00000080:3.0:1773941420.797120:0:12100:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0xa33a5be14eae1c21 00000020:00000040:3.0:1773941420.797123:0:12100:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b88956800 with handle 0xa33a5be14eae1c21 from hash 00000020:00000001:3.0:1773941420.797126:0:12100:0:(genops.c:1934:obd_stale_export_put()) Process entered 00000020:00000040:3.0:1773941420.797128:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b88956800 : new refcount 1 00000020:00000001:3.0:1773941420.797134:0:12100:0:(genops.c:1959:obd_stale_export_put()) Process leaving 00000020:00000040:3.0:1773941420.797135:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b88956800 : new refcount 0 00000020:00000080:3.0:1773941420.797136:0:12100:0:(genops.c:904:class_export_put()) final put ffff8b5b88956800/637f784d-29a5-48a3-b831-1c4c7c7a2a54 00000020:00000001:3.0:1773941420.797138:0:12100:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.797149:0:12100:0:(genops.c:1493:class_disconnect()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.797150:0:12100:0:(ldlm_lib.c:831:client_disconnect_export()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:3.0:1773941420.797153:0:12100:0:(obd_class.h:914:obd_disconnect()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:3.0:1773941420.797154:0:12100:0:(lmv_obd.c:579:lmv_disconnect_mdc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.797166:0:12100:0:(genops.c:1462:class_disconnect()) Process entered 00000020:00000080:3.0:1773941420.797167:0:12100:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0xa33a5be14eae1c1a 00000020:00000040:3.0:1773941420.797168:0:12100:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b88954800 with handle 0xa33a5be14eae1c1a from hash 00000020:00000001:3.0:1773941420.797171:0:12100:0:(genops.c:1934:obd_stale_export_put()) Process entered 00000020:00000040:3.0:1773941420.797172:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b88954800 : new refcount 1 00000020:00000001:0.0:1773941420.797172:0:36:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000001:3.0:1773941420.797176:0:12100:0:(genops.c:1959:obd_stale_export_put()) Process leaving 00000020:00000080:0.0:1773941420.797177:0:36:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b88956800/637f784d-29a5-48a3-b831-1c4c7c7a2a54 for lustre-MDT0000-mdc-ffff8b5b885e5800 00000020:00000040:3.0:1773941420.797178:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b88954800 : new refcount 0 00000020:00000080:3.0:1773941420.797181:0:12100:0:(genops.c:904:class_export_put()) final put ffff8b5b88954800/637f784d-29a5-48a3-b831-1c4c7c7a2a54 00000020:00000040:0.0:1773941420.797184:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 5 to 192.168.204.143@tcp 00000020:00000001:3.0:1773941420.797185:0:12100:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941420.797189:0:36:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:0.0:1773941420.797191:0:36:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941420.797193:0:36:0:(obd_config.c:1058:class_decref()) Decref lustre-MDT0000-mdc-ffff8b5b885e5800 (ffff8b5b855119b8) now 5 - export 00000020:00000001:3.0:1773941420.797196:0:12100:0:(genops.c:1493:class_disconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941420.797196:0:36:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff8b5b88956800. 00800000:00000001:3.0:1773941420.797201:0:12100:0:(lmv_obd.c:600:lmv_disconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941420.797203:0:36:0:(genops.c:880:class_export_destroy()) Process leaving 00000080:00000001:3.0:1773941420.797204:0:12100:0:(obd_class.h:914:obd_disconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1773941420.797207:0:12100:0:(llite_lib.c:995:client_common_put_super()) Process leaving 00000020:00000001:0.0:1773941420.797208:0:36:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000001:3.0:1773941420.797211:0:12100:0:(obd_config.c:2489:class_manual_cleanup()) Process entered 00000020:00000080:0.0:1773941420.797211:0:36:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b88954800/637f784d-29a5-48a3-b831-1c4c7c7a2a54 for lustre-clilmv-ffff8b5b885e5800 00000020:01000000:3.0:1773941420.797214:0:12100:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-clilov-ffff8b5b885e5800 (flags='') 00000020:00000010:3.0:1773941420.797217:0:12100:0:(obd_config.c:2506:class_manual_cleanup()) kmalloced '(lcfg)': 80 at ffff8b5ba0bc9a20. 00000020:00000080:3.0:1773941420.797220:0:12100:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000001:0.0:1773941420.797220:0:36:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:0.0:1773941420.797222:0:36:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.797225:0:12100:0:(obd_config.c:936:class_cleanup()) Process entered 00000020:00000040:0.0:1773941420.797225:0:36:0:(obd_config.c:1058:class_decref()) Decref lustre-clilmv-ffff8b5b885e5800 (ffff8b5b85510000) now 4 - export 00000020:00000080:3.0:1773941420.797229:0:12100:0:(obd_config.c:991:class_cleanup()) lustre-clilov-ffff8b5b885e5800: forcing exports to disconnect: 0/1 00000020:00000010:0.0:1773941420.797229:0:36:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff8b5b88954800. 00000020:00000001:3.0:1773941420.797234:0:12100:0:(genops.c:1562:class_disconnect_exports()) Process entered 00000020:00000001:0.0:1773941420.797235:0:36:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00080000:3.0:1773941420.797236:0:12100:0:(genops.c:1577:class_disconnect_exports()) OBD device 1 (ffff8b5b85513370) has no exports 00000020:00000001:3.0:1773941420.797239:0:12100:0:(genops.c:1578:class_disconnect_exports()) Process leaving 00000020:00000001:3.0:1773941420.797241:0:12100:0:(obd_class.h:604:obd_precleanup()) Process entered 00000020:00000001:3.0:1773941420.797242:0:12100:0:(obd_class.h:609:obd_precleanup()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.797254:0:12100:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:3.0:1773941420.797258:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5ba0f90c00. 00000001:00000010:3.0:1773941420.797260:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5ba0f91e00. 00000001:00000010:3.0:1773941420.797276:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5ba0f91000. 00000001:00000010:3.0:1773941420.797278:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5ba0f91800. 00000001:00000010:3.0:1773941420.797280:0:12100:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 32 at ffff8b5b88570040. 00000001:00000010:3.0:1773941420.797282:0:12100:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 232 at ffff8b5ba0c22800. 00000001:00000001:3.0:1773941420.797284:0:12100:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00000020:00000040:3.0:1773941420.797285:0:12100:0:(obd_config.c:1058:class_decref()) Decref lustre-clilov-ffff8b5b885e5800 (ffff8b5b85513370) now 3 - setup 00000020:00000001:3.0:1773941420.797287:0:12100:0:(obd_config.c:1021:class_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.797288:0:12100:0:(obd_config.c:1659:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000080:3.0:1773941420.797290:0:12100:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000001:3.0:1773941420.797292:0:12100:0:(obd_config.c:899:class_detach()) Process entered 00000020:00000040:3.0:1773941420.797294:0:12100:0:(obd_config.c:1058:class_decref()) Decref lustre-clilov-ffff8b5b885e5800 (ffff8b5b85513370) now 2 - obd_device_list 00000020:00000080:3.0:1773941420.797296:0:12100:0:(obd_config.c:919:class_detach()) detach on obd lustre-clilov-ffff8b5b885e5800 (uuid 637f784d-29a5-48a3-b831-1c4c7c7a2a54) 00000020:00000040:3.0:1773941420.797297:0:12100:0:(obd_config.c:1058:class_decref()) Decref lustre-clilov-ffff8b5b885e5800 (ffff8b5b85513370) now 1 - newdev 00000020:00000040:3.0:1773941420.797299:0:12100:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b88951000 with handle 0xa33a5be14eae1bf0 from hash 00000020:00000040:3.0:1773941420.797301:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b88951000 : new refcount 0 00000020:00000080:3.0:1773941420.797302:0:12100:0:(genops.c:904:class_export_put()) final put ffff8b5b88951000/637f784d-29a5-48a3-b831-1c4c7c7a2a54 00000020:00000001:3.0:1773941420.797303:0:12100:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.797304:0:12100:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:3.0:1773941420.797305:0:12100:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b88951000/637f784d-29a5-48a3-b831-1c4c7c7a2a54 for lustre-clilov-ffff8b5b885e5800 00000020:00000001:3.0:1773941420.797306:0:12100:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:3.0:1773941420.797307:0:12100:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.797309:0:12100:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff8b5b88951000. 00000020:00000001:3.0:1773941420.797311:0:12100:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000040:3.0:1773941420.797313:0:12100:0:(genops.c:437:class_free_dev()) Release obd device lustre-clilov-ffff8b5b885e5800 obd_type name = lov 00000020:01000000:3.0:1773941420.797314:0:12100:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-clilov-ffff8b5b885e5800 (637f784d-29a5-48a3-b831-1c4c7c7a2a54) 00000020:00000001:3.0:1773941420.797315:0:12100:0:(obd_class.h:633:obd_cleanup()) Process entered 00000020:00000001:3.0:1773941420.797316:0:12100:0:(obd_class.h:638:obd_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.797317:0:12100:0:(genops.c:66:obd_device_free()) slab-freed '(obd)': 6584 at ffff8b5b85513370. 00000020:00000001:3.0:1773941420.797322:0:12100:0:(obd_config.c:923:class_detach()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.797323:0:12100:0:(obd_config.c:1655:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1773941420.797325:0:12100:0:(obd_config.c:2523:class_manual_cleanup()) kfreed 'lcfg': 80 at ffff8b5ba0bc9a20. 00000020:00000001:3.0:1773941420.797326:0:12100:0:(obd_config.c:2524:class_manual_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.797331:0:12100:0:(obd_config.c:2489:class_manual_cleanup()) Process entered 00000020:01000000:3.0:1773941420.797332:0:12100:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-clilmv-ffff8b5b885e5800 (flags='') 00000020:00000010:3.0:1773941420.797333:0:12100:0:(obd_config.c:2506:class_manual_cleanup()) kmalloced '(lcfg)': 80 at ffff8b5ba0bc9a20. 00000020:00000080:3.0:1773941420.797334:0:12100:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000001:3.0:1773941420.797336:0:12100:0:(obd_config.c:936:class_cleanup()) Process entered 00000020:00000080:3.0:1773941420.797337:0:12100:0:(obd_config.c:991:class_cleanup()) lustre-clilmv-ffff8b5b885e5800: forcing exports to disconnect: 0/1 00000020:00000001:3.0:1773941420.797338:0:12100:0:(genops.c:1562:class_disconnect_exports()) Process entered 00000020:00080000:3.0:1773941420.797339:0:12100:0:(genops.c:1577:class_disconnect_exports()) OBD device 2 (ffff8b5b85510000) has no exports 00000020:00000001:3.0:1773941420.797340:0:12100:0:(genops.c:1578:class_disconnect_exports()) Process leaving 00000020:00000001:3.0:1773941420.797341:0:12100:0:(obd_class.h:604:obd_precleanup()) Process entered 00000020:00000010:3.0:1773941420.797344:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84e8e400. 00000020:00000010:3.0:1773941420.797346:0:12100:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b885b0400. 00000040:00000010:3.0:1773941420.797348:0:12100:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b885b1600. 00000020:00000010:3.0:1773941420.797355:0:12100:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b98ac3000. 00000020:00000010:3.0:1773941420.797359:0:12100:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab9200. 00040000:00000010:3.0:1773941420.797362:0:12100:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98a69900. 00040000:00000010:3.0:1773941420.797366:0:12100:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b8809ca00. 00000020:00000010:3.0:1773941420.797369:0:12100:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068140. 00000020:00000001:3.0:1773941420.797372:0:12100:0:(kernelcomm.c:1231:libcfs_kkuc_group_rem()) Process entered 00000020:00000001:3.0:1773941420.797374:0:12100:0:(kernelcomm.c:1272:libcfs_kkuc_group_put()) Process entered 00000020:00000001:3.0:1773941420.797375:0:12100:0:(kernelcomm.c:1285:libcfs_kkuc_group_put()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000020:00000001:3.0:1773941420.797377:0:12100:0:(kernelcomm.c:1263:libcfs_kkuc_group_rem()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.797380:0:12100:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068140. 00000020:00000010:3.0:1773941420.797383:0:12100:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b885b0400. 00000040:00000010:3.0:1773941420.797385:0:12100:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b885b1600. 00000020:00000010:3.0:1773941420.797386:0:12100:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b98ac3000. 00000020:00000010:3.0:1773941420.797390:0:12100:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab9200. 00040000:00000010:3.0:1773941420.797392:0:12100:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98a69900. 00040000:00000010:3.0:1773941420.797394:0:12100:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b8809ca00. 00000020:00000010:3.0:1773941420.797396:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84e8e400. 00000020:00000001:3.0:1773941420.797397:0:12100:0:(obd_class.h:623:obd_precleanup()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.797406:0:12100:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:3.0:1773941420.797409:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5b91ecae00. 00000001:00000010:3.0:1773941420.797411:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5b91ecbc00. 00000001:00000010:3.0:1773941420.797412:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5b91eca600. 00000001:00000010:3.0:1773941420.797418:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5b91ecb800. 00000001:00000010:3.0:1773941420.797419:0:12100:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 32 at ffff8b5b88570ee0. 00000001:00000010:3.0:1773941420.797421:0:12100:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 232 at ffff8b5ba0c22500. 00000001:00000001:3.0:1773941420.797423:0:12100:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00000020:00000040:3.0:1773941420.797424:0:12100:0:(obd_config.c:1058:class_decref()) Decref lustre-clilmv-ffff8b5b885e5800 (ffff8b5b85510000) now 3 - setup 00000020:00000001:3.0:1773941420.797425:0:12100:0:(obd_config.c:1021:class_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.797426:0:12100:0:(obd_config.c:1659:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000080:3.0:1773941420.797427:0:12100:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000001:3.0:1773941420.797428:0:12100:0:(obd_config.c:899:class_detach()) Process entered 00000020:00000040:3.0:1773941420.797430:0:12100:0:(obd_config.c:1058:class_decref()) Decref lustre-clilmv-ffff8b5b885e5800 (ffff8b5b85510000) now 2 - obd_device_list 00000020:00000080:3.0:1773941420.797431:0:12100:0:(obd_config.c:919:class_detach()) detach on obd lustre-clilmv-ffff8b5b885e5800 (uuid 637f784d-29a5-48a3-b831-1c4c7c7a2a54) 00000020:00000040:3.0:1773941420.797433:0:12100:0:(obd_config.c:1058:class_decref()) Decref lustre-clilmv-ffff8b5b885e5800 (ffff8b5b85510000) now 1 - newdev 00000020:00000040:3.0:1773941420.797434:0:12100:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b88950800 with handle 0xa33a5be14eae1bf7 from hash 00000020:00000040:3.0:1773941420.797436:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b88950800 : new refcount 0 00000020:00000080:3.0:1773941420.797437:0:12100:0:(genops.c:904:class_export_put()) final put ffff8b5b88950800/637f784d-29a5-48a3-b831-1c4c7c7a2a54 00000020:00000001:3.0:1773941420.797439:0:12100:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.797440:0:12100:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:3.0:1773941420.797441:0:12100:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b88950800/637f784d-29a5-48a3-b831-1c4c7c7a2a54 for lustre-clilmv-ffff8b5b885e5800 00000020:00000001:3.0:1773941420.797442:0:12100:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:3.0:1773941420.797443:0:12100:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.797444:0:12100:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff8b5b88950800. 00000020:00000001:3.0:1773941420.797447:0:12100:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000040:3.0:1773941420.797448:0:12100:0:(genops.c:437:class_free_dev()) Release obd device lustre-clilmv-ffff8b5b885e5800 obd_type name = lmv 00000020:01000000:3.0:1773941420.797449:0:12100:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-clilmv-ffff8b5b885e5800 (637f784d-29a5-48a3-b831-1c4c7c7a2a54) 00000020:00000001:3.0:1773941420.797450:0:12100:0:(obd_class.h:633:obd_cleanup()) Process entered 00000020:00000010:3.0:1773941420.797453:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b8809ca00. 00000020:00000010:3.0:1773941420.797455:0:12100:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b8809c800. 00000040:00000010:3.0:1773941420.797457:0:12100:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b8809da00. 00000020:00000010:3.0:1773941420.797460:0:12100:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b98ac3000. 00000020:00000010:3.0:1773941420.797462:0:12100:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b8809d400. 00040000:00000010:3.0:1773941420.797464:0:12100:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98a69900. 00040000:00000010:3.0:1773941420.797466:0:12100:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b8809ce00. 00800000:00000001:3.0:1773941420.797471:0:12100:0:(lmv_obd.c:1313:lmv_device_free()) Process entered 00800000:00000010:3.0:1773941420.797473:0:12100:0:(lmv_obd.c:1319:lmv_device_free()) kfreed 'pat': 280 at ffff8b5ba0c90800. 00800000:00000010:3.0:1773941420.797475:0:12100:0:(lmv_obd.c:1319:lmv_device_free()) kfreed 'pat': 280 at ffff8b5ba0c90200. 80000000:00000001:3.0:1773941420.797477:0:12100:0:(fld_request.c:280:fld_client_fini()) Process entered 80000000:00000001:3.0:1773941420.797479:0:12100:0:(fld_cache.c:227:fld_cache_flush()) Process entered 80000000:00000001:3.0:1773941420.797481:0:12100:0:(fld_cache.c:198:fld_cache_shrink()) Process entered 80000000:00000040:3.0:1773941420.797482:0:12100:0:(fld_cache.c:217:fld_cache_shrink()) cli-lustre-clilmv-ffff8b5b885e5800: FLD cache - Shrunk by 0 entries 80000000:00000001:3.0:1773941420.797484:0:12100:0:(fld_cache.c:219:fld_cache_shrink()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:3.0:1773941420.797485:0:12100:0:(fld_cache.c:234:fld_cache_flush()) Process leaving 80000000:00000040:3.0:1773941420.797486:0:12100:0:(fld_cache.c:80:fld_cache_fini()) FLD cache statistics (cli-lustre-clilmv-ffff8b5b885e5800): 80000000:00000040:3.0:1773941420.797487:0:12100:0:(fld_cache.c:81:fld_cache_fini()) Cache reqs: 0 80000000:00000040:3.0:1773941420.797488:0:12100:0:(fld_cache.c:82:fld_cache_fini()) Total reqs: 0 80000000:00000010:3.0:1773941420.797489:0:12100:0:(fld_cache.c:84:fld_cache_fini()) kfreed 'cache': 168 at ffff8b5b885de240. 80000000:00000001:3.0:1773941420.797492:0:12100:0:(fld_request.c:298:fld_client_fini()) Process leaving 00000020:00000010:3.0:1773941420.797582:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 640 at ffff8b5ba0735800. 00000020:00000010:3.0:1773941420.797584:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 640 at ffff8b5b83f0a000. 00000020:00000010:3.0:1773941420.797587:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 512 at ffff8b5ba0c91000. 00000020:00000010:3.0:1773941420.797589:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5ba0ce4400. 00020000:00000001:3.0:1773941420.797592:0:12100:0:(lu_tgt_descs.c:164:lu_qos_del_tgt()) Process entered 00020000:00000001:3.0:1773941420.797594:0:12100:0:(lu_tgt_descs.c:169:lu_qos_del_tgt()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00020000:00000001:3.0:1773941420.797595:0:12100:0:(lu_tgt_descs.c:186:lu_qos_del_tgt()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00800000:00000010:3.0:1773941420.797597:0:12100:0:(lmv_obd.c:410:lmv_del_target()) kfreed 'tgt': 368 at ffff8b5ba0f91a00. 00020000:00000010:3.0:1773941420.797599:0:12100:0:(lu_tgt_descs.c:285:lu_tgt_descs_fini()) kfreed 'ltd->ltd_tgt_idx[i]': 4096 at ffff8b5b90f31000. 00800000:00000010:3.0:1773941420.797602:0:12100:0:(lmv_obd.c:1331:lmv_device_free()) kfreed 'lu': 64 at ffff8b5b82c59280. 00800000:00000001:3.0:1773941420.797603:0:12100:0:(lmv_obd.c:1333:lmv_device_free()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.797605:0:12100:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b8809c800. 00000040:00000010:3.0:1773941420.797606:0:12100:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b8809da00. 00000020:00000010:3.0:1773941420.797608:0:12100:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b98ac3000. 00000020:00000010:3.0:1773941420.797610:0:12100:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b8809d400. 00040000:00000010:3.0:1773941420.797611:0:12100:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98a69900. 00040000:00000010:3.0:1773941420.797612:0:12100:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b8809ce00. 00000020:00000010:3.0:1773941420.797614:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b8809ca00. 00000020:00000001:3.0:1773941420.797615:0:12100:0:(obd_class.h:647:obd_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.797616:0:12100:0:(genops.c:66:obd_device_free()) slab-freed '(obd)': 6584 at ffff8b5b85510000. 00000020:00000001:3.0:1773941420.797619:0:12100:0:(obd_config.c:923:class_detach()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.797625:0:12100:0:(obd_config.c:1655:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1773941420.797626:0:12100:0:(obd_config.c:2523:class_manual_cleanup()) kfreed 'lcfg': 80 at ffff8b5ba0bc9a20. 00000020:00000001:3.0:1773941420.797628:0:12100:0:(obd_config.c:2524:class_manual_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.797629:0:12100:0:(obd_config.c:2489:class_manual_cleanup()) Process entered 00000020:01000000:3.0:1773941420.797631:0:12100:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-MDT0000-mdc-ffff8b5b885e5800 (flags='') 00000020:00000010:3.0:1773941420.797633:0:12100:0:(obd_config.c:2506:class_manual_cleanup()) kmalloced '(lcfg)': 88 at ffff8b5ba0bc9a20. 00000020:00000080:3.0:1773941420.797634:0:12100:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000001:3.0:1773941420.797636:0:12100:0:(obd_config.c:936:class_cleanup()) Process entered 00000020:00000080:3.0:1773941420.797637:0:12100:0:(obd_config.c:991:class_cleanup()) lustre-MDT0000-mdc-ffff8b5b885e5800: forcing exports to disconnect: 0/2 00000020:00000001:3.0:1773941420.797639:0:12100:0:(genops.c:1562:class_disconnect_exports()) Process entered 00000020:00080000:3.0:1773941420.797640:0:12100:0:(genops.c:1577:class_disconnect_exports()) OBD device 3 (ffff8b5b855119b8) has no exports 00000020:00000001:3.0:1773941420.797641:0:12100:0:(genops.c:1578:class_disconnect_exports()) Process leaving 00000020:00000001:3.0:1773941420.797642:0:12100:0:(obd_class.h:604:obd_precleanup()) Process entered 00000020:00000010:3.0:1773941420.797643:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b8809ca00. 00000020:00000010:3.0:1773941420.797645:0:12100:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b8809ce00. 00000020:00000010:3.0:1773941420.797647:0:12100:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc91e0. 00008000:00000010:3.0:1773941420.797649:0:12100:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b98a1e6d8. 00000020:00000010:3.0:1773941420.797650:0:12100:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b850689c0. 00000002:00000001:3.0:1773941420.797653:0:12100:0:(mdc_dev.c:1760:mdc_device_fini()) Process entered 00000008:00000001:3.0:1773941420.797654:0:12100:0:(osc_request.c:4123:osc_precleanup_common()) Process entered 00000008:00000001:3.0:1773941420.797658:0:12100:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 00000008:01000000:3.0:1773941420.797659:0:12100:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-MDT0000-mdc-ffff8b5b885e5800: client import never connected 00000100:00000001:3.0:1773941420.797660:0:12100:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941420.797661:0:12100:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0000_UUID INVALID 00000100:00000001:3.0:1773941420.797662:0:12100:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:3.0:1773941420.797663:0:12100:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00100000:3.0:1773941420.797665:0:12100:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b5b885e5800: committing for last_committed 8589934593 gen 3 00000100:00000040:3.0:1773941420.797666:0:12100:0:(client.c:3191:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b5b885e5800: normal: skip=0 replay=0/0 committed=0/0 00000100:00000001:3.0:1773941420.797667:0:12100:0:(client.c:3192:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1773941420.797668:0:12100:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941420.797669:0:12100:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941420.797670:0:12100:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:3.0:1773941420.797671:0:12100:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000002:00000001:3.0:1773941420.797672:0:12100:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.797673:0:12100:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00100000:3.0:1773941420.797679:0:12100:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941420.797680:0:12100:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:3.0:1773941420.797681:0:12100:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000001:00000001:3.0:1773941420.797682:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.797683:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.797984:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.797986:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.797987:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.798254:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.798256:0:12100:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1773941420.798258:0:12100:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8b5b850f85a0. 00000020:00000010:3.0:1773941420.798260:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b8809d400. 00000020:00000010:3.0:1773941420.798262:0:12100:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b8809da00. 00000020:00000010:3.0:1773941420.798263:0:12100:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9f00. 00008000:00000010:3.0:1773941420.798276:0:12100:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b98a1d1a8. 00000020:00000010:3.0:1773941420.798279:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b8809c800. 00008000:00000010:3.0:1773941420.798281:0:12100:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5b98a6ba30. 00000020:00000001:3.0:1773941420.798282:0:12100:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446615823923512744 : -128249786038872 : ffff8b5b850f85a8) 00000020:00001000:3.0:1773941420.798284:0:12100:0:(cl_object.c:831:cl_env_get()) 1@ffff8b5b850f85a0 00000008:00000001:3.0:1773941420.798285:0:12100:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1773941420.798286:0:12100:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1773941420.798287:0:12100:0:(osc_cache.c:2316:osc_next_obj()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1773941420.798288:0:12100:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000001:00000001:3.0:1773941420.798289:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.798290:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.798558:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:3.0:1773941420.798559:0:12100:0:(cl_object.c:925:cl_env_put()) 1@ffff8b5b850f85a0 00000020:00000010:3.0:1773941420.798561:0:12100:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b8809da00. 00000020:00000010:3.0:1773941420.798563:0:12100:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9f00. 00008000:00000010:3.0:1773941420.798565:0:12100:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b98a1d1a8. 00000020:00000010:3.0:1773941420.798567:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b8809d400. 00008000:00000010:3.0:1773941420.798569:0:12100:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8b5b98a6ba30. 00000020:00000010:3.0:1773941420.798571:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b8809c800. 00000020:00000010:3.0:1773941420.798572:0:12100:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8b5b850f85a0. 00000001:00000001:3.0:1773941420.798574:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.798575:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.798873:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.798880:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.798881:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.799191:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:3.0:1773941420.799194:0:12100:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.799196:0:12100:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:3.0:1773941420.799200:0:12100:0:(genops.c:1229:class_import_get()) import ffff8b5b88953800 refcount=3 obd=lustre-MDT0000-mdc-ffff8b5b885e5800 00000020:00000001:3.0:1773941420.799203:0:12100:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941420.799204:0:12100:0:(genops.c:1242:class_import_put()) import ffff8b5b88953800 refcount=2 obd=lustre-MDT0000-mdc-ffff8b5b885e5800 00000020:00000001:3.0:1773941420.799206:0:12100:0:(genops.c:1249:class_import_put()) Process leaving 00000020:00000001:3.0:1773941420.799207:0:12100:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941420.799209:0:12100:0:(genops.c:1242:class_import_put()) import ffff8b5b88953800 refcount=1 obd=lustre-MDT0000-mdc-ffff8b5b885e5800 00000020:00000001:3.0:1773941420.799211:0:12100:0:(genops.c:1249:class_import_put()) Process leaving 00000008:00000001:3.0:1773941420.799213:0:12100:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 00000008:00000001:3.0:1773941420.799215:0:12100:0:(osc_request.c:4138:osc_precleanup_common()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:3.0:1773941420.799217:0:12100:0:(mdc_changelog.c:886:mdc_changelog_cdev_finish()) Process entered 00000002:00000001:3.0:1773941420.799220:0:12100:0:(mdc_changelog.c:136:chlg_dev_clear()) Process entered 00000002:00000010:3.0:1773941420.799422:0:12100:0:(mdc_changelog.c:128:chlg_device_release()) kfreed 'entry': 1304 at ffff8b5b88951800. 00000002:00000001:3.0:1773941420.799427:0:12100:0:(mdc_changelog.c:143:chlg_dev_clear()) Process leaving 00000002:00000001:3.0:1773941420.799429:0:12100:0:(mdc_changelog.c:892:mdc_changelog_cdev_finish()) Process leaving 00000002:00000001:3.0:1773941420.799431:0:12100:0:(mdc_request.c:3022:mdc_llog_finish()) Process entered 00000002:00000040:3.0:1773941420.799433:0:12100:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5b885de540 : new refcount 2 00000040:00000001:3.0:1773941420.799436:0:12100:0:(llog_obd.c:93:llog_cleanup()) Process entered 00000040:00000040:3.0:1773941420.799438:0:12100:0:(lustre_log.h:400:llog_ctxt_put()) PUTting ctxt ffff8b5b885de540 : new refcount 1 00000020:00000040:3.0:1773941420.799441:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b88950000 : new refcount 1 00000020:00000001:3.0:1773941420.799442:0:12100:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941420.799443:0:12100:0:(genops.c:1242:class_import_put()) import ffff8b5b88953800 refcount=0 obd=lustre-MDT0000-mdc-ffff8b5b885e5800 00000020:00000040:3.0:1773941420.799446:0:12100:0:(genops.c:1245:class_import_put()) final put import ffff8b5b88953800 00000020:00000001:3.0:1773941420.799455:0:12100:0:(genops.c:1249:class_import_put()) Process leaving 00000040:00000010:3.0:1773941420.799457:0:12100:0:(llog_obd.c:46:llog_ctxt_destroy()) kfreed 'ctxt': 168 at ffff8b5b885de540. 00000040:00000001:3.0:1773941420.799460:0:12100:0:(llog_obd.c:122:llog_cleanup()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:3.0:1773941420.799461:0:12100:0:(mdc_request.c:3027:mdc_llog_finish()) Process leaving 00000020:00000010:3.0:1773941420.799463:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 640 at ffff8b5ba0737800. 00000020:00000010:3.0:1773941420.799465:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 640 at ffff8b5b83f08800. 00000020:00000010:3.0:1773941420.799467:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 512 at ffff8b5ba0c91e00. 00000020:00000010:3.0:1773941420.799468:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5ba0c9c000. 00000020:00000001:2.0:1773941420.799481:0:36:0:(genops.c:1200:obd_zombie_import_free()) Process entered 00000020:00000080:2.0:1773941420.799485:0:36:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff8b5b88953800 for lustre-MDT0000-mdc-ffff8b5b885e5800 00000020:00000040:2.0:1773941420.799496:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 4 to 192.168.204.143@tcp 00000020:00000040:2.0:1773941420.799501:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000020:00000010:2.0:1773941420.799504:0:36:0:(genops.c:1213:obd_zombie_import_free()) kfreed 'imp_conn': 88 at ffff8b5b84cfb8a0. 00000020:00000040:2.0:1773941420.799508:0:36:0:(obd_config.c:1058:class_decref()) Decref lustre-MDT0000-mdc-ffff8b5b885e5800 (ffff8b5b855119b8) now 4 - import 00000020:00000010:2.0:1773941420.799511:0:36:0:(genops.c:1220:obd_zombie_import_free()) kfreed 'imp': 1712 at ffff8b5b88953800. 00000020:00000001:2.0:1773941420.799515:0:36:0:(genops.c:1221:obd_zombie_import_free()) Process leaving 00000020:00000010:3.0:1773941420.799667:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff8b5b850dc000. 00000020:00000010:3.0:1773941420.799678:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff8b5b85e54000. 00000020:00000010:3.0:1773941420.799681:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff8b5ba0c74000. 00000020:00000010:3.0:1773941420.799685:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff8b5b90d44000. 00000020:00000010:3.0:1773941420.799689:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 3680 at ffff8b5b85e5d000. 00000020:00000010:3.0:1773941420.799693:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5ba0c9f000. 00000002:00000001:3.0:1773941420.799695:0:12100:0:(mdc_dev.c:1768:mdc_device_fini()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.799699:0:12100:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b850689c0. 00000020:00000010:3.0:1773941420.799736:0:12100:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b8809ce00. 00000020:00000010:3.0:1773941420.799740:0:12100:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc91e0. 00008000:00000010:3.0:1773941420.799745:0:12100:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b98a1e6d8. 00000020:00000010:3.0:1773941420.799750:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b8809ca00. 00000020:00000001:3.0:1773941420.799753:0:12100:0:(obd_class.h:623:obd_precleanup()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.799767:0:12100:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:3.0:1773941420.799772:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5ba0c90a00. 00000001:00000010:3.0:1773941420.799775:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5ba0c91400. 00000001:00000010:3.0:1773941420.799778:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5ba0c90400. 00000001:00000010:3.0:1773941420.799780:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5ba0c91c00. 00000001:00000010:3.0:1773941420.799782:0:12100:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 32 at ffff8b5b88570800. 00000001:00000010:3.0:1773941420.799784:0:12100:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 232 at ffff8b5ba0c22c00. 00000001:00000001:3.0:1773941420.799788:0:12100:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00000020:00000040:3.0:1773941420.799790:0:12100:0:(obd_config.c:1058:class_decref()) Decref lustre-MDT0000-mdc-ffff8b5b885e5800 (ffff8b5b855119b8) now 3 - setup 00000020:00000001:3.0:1773941420.799792:0:12100:0:(obd_config.c:1021:class_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.799795:0:12100:0:(obd_config.c:1659:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000080:3.0:1773941420.799797:0:12100:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000001:3.0:1773941420.799801:0:12100:0:(obd_config.c:899:class_detach()) Process entered 00000020:00000040:3.0:1773941420.799806:0:12100:0:(obd_config.c:1058:class_decref()) Decref lustre-MDT0000-mdc-ffff8b5b885e5800 (ffff8b5b855119b8) now 2 - obd_device_list 00000020:00000080:3.0:1773941420.799808:0:12100:0:(obd_config.c:919:class_detach()) detach on obd lustre-MDT0000-mdc-ffff8b5b885e5800 (uuid 637f784d-29a5-48a3-b831-1c4c7c7a2a54) 00000020:00000040:3.0:1773941420.799810:0:12100:0:(obd_config.c:1058:class_decref()) Decref lustre-MDT0000-mdc-ffff8b5b885e5800 (ffff8b5b855119b8) now 1 - newdev 00000020:00000040:3.0:1773941420.799813:0:12100:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b88950000 with handle 0xa33a5be14eae1bfe from hash 00000020:00000040:3.0:1773941420.799815:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b88950000 : new refcount 0 00000020:00000080:3.0:1773941420.799817:0:12100:0:(genops.c:904:class_export_put()) final put ffff8b5b88950000/637f784d-29a5-48a3-b831-1c4c7c7a2a54 00000020:00000001:3.0:1773941420.799819:0:12100:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.799826:0:12100:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:3.0:1773941420.799827:0:12100:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b88950000/637f784d-29a5-48a3-b831-1c4c7c7a2a54 for lustre-MDT0000-mdc-ffff8b5b885e5800 00000020:00000001:3.0:1773941420.799829:0:12100:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:3.0:1773941420.799830:0:12100:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.799831:0:12100:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff8b5b88950000. 00000020:00000001:3.0:1773941420.799835:0:12100:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000040:3.0:1773941420.799836:0:12100:0:(genops.c:437:class_free_dev()) Release obd device lustre-MDT0000-mdc-ffff8b5b885e5800 obd_type name = mdc 00000020:01000000:3.0:1773941420.799837:0:12100:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-MDT0000-mdc-ffff8b5b885e5800 (637f784d-29a5-48a3-b831-1c4c7c7a2a54) 00000020:00000001:3.0:1773941420.799838:0:12100:0:(obd_class.h:633:obd_cleanup()) Process entered 00000020:00000010:3.0:1773941420.799841:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b8809ca00. 00000020:00000010:3.0:1773941420.799843:0:12100:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b8809ce00. 00000020:00000010:3.0:1773941420.799846:0:12100:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc91e0. 00008000:00000010:3.0:1773941420.799848:0:12100:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b98a1e6d8. 00000008:00000001:3.0:1773941420.799851:0:12100:0:(osc_request.c:4146:osc_cleanup_common()) Process entered 00010000:00000001:3.0:1773941420.799854:0:12100:0:(ldlm_lib.c:631:client_obd_cleanup()) Process entered 00010000:00000001:3.0:1773941420.799856:0:12100:0:(ldlm_resource.c:1304:ldlm_namespace_free_post()) Process entered 00010000:00000001:3.0:1773941420.799858:0:12100:0:(ldlm_pool.c:931:ldlm_pool_fini()) Process entered 00000020:00000010:3.0:1773941420.799885:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 448 at ffff8b5b902a1400. 00000020:00000010:3.0:1773941420.799887:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 448 at ffff8b5b89317a00. 00000020:00000010:3.0:1773941420.799889:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 448 at ffff8b5b885b0600. 00000020:00000010:3.0:1773941420.799891:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 352 at ffff8b5ba0c91800. 00000020:00000010:3.0:1773941420.799894:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5ba0c9f400. 00010000:00000001:3.0:1773941420.799928:0:12100:0:(ldlm_pool.c:934:ldlm_pool_fini()) Process leaving 00000020:00000010:3.0:1773941420.799936:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff8b5b90edba80. 00000020:00000010:3.0:1773941420.799938:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff8b5b89390500. 00000020:00000010:3.0:1773941420.799939:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff8b5b9020a200. 00000020:00000010:3.0:1773941420.799941:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 96 at ffff8b5b880f9720. 00000020:00000010:3.0:1773941420.799943:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5ba0c9e000. 00000001:00000001:3.0:1773941420.799979:0:12100:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:3.0:1773941420.800238:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41cd5000. 00000001:00000010:3.0:1773941420.800246:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41cdf000. 00000001:00000010:3.0:1773941420.800247:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41ce9000. 00000001:00000010:3.0:1773941420.800249:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41cf3000. 00000001:00000010:3.0:1773941420.800255:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41cfd000. 00000001:00000010:3.0:1773941420.800256:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41d07000. 00000001:00000010:3.0:1773941420.800258:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41d11000. 00000001:00000010:3.0:1773941420.800259:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41d1b000. 00000001:00000010:3.0:1773941420.800260:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41d25000. 00000001:00000010:3.0:1773941420.800262:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41d2f000. 00000001:00000010:3.0:1773941420.800263:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41d39000. 00000001:00000010:3.0:1773941420.800264:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41d4d000. 00000001:00000010:3.0:1773941420.800265:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41d57000. 00000001:00000010:3.0:1773941420.800278:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41d61000. 00000001:00000010:3.0:1773941420.800280:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41d6b000. 00000001:00000010:3.0:1773941420.800280:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41d75000. 00000001:00000010:3.0:1773941420.800281:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41d7f000. 00000001:00000010:3.0:1773941420.800282:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41d89000. 00000001:00000010:3.0:1773941420.800284:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41d93000. 00000001:00000010:3.0:1773941420.800284:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41d9d000. 00000001:00000010:3.0:1773941420.800285:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41dad000. 00000001:00000010:3.0:1773941420.800287:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41dbd000. 00000001:00000010:3.0:1773941420.800288:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41dc7000. 00000001:00000010:3.0:1773941420.800289:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41dd1000. 00000001:00000010:3.0:1773941420.800291:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41ddb000. 00000001:00000010:3.0:1773941420.800292:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41de5000. 00000001:00000010:3.0:1773941420.800293:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41df5000. 00000001:00000010:3.0:1773941420.800294:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41e1d000. 00000001:00000010:3.0:1773941420.800295:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41e27000. 00000001:00000010:3.0:1773941420.800296:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41e31000. 00000001:00000010:3.0:1773941420.800297:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41e3b000. 00000001:00000010:3.0:1773941420.800298:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffa10c41e45000. 00000001:00000010:3.0:1773941420.800299:0:12100:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 256 at ffff8b5ba0c22100. 00000001:00000010:3.0:1773941420.800302:0:12100:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 280 at ffff8b5ba0c90000. 00000001:00000001:3.0:1773941420.800303:0:12100:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00010000:00000010:3.0:1773941420.800305:0:12100:0:(ldlm_resource.c:1327:ldlm_namespace_free_post()) kfreed 'ns': 848 at ffff8b5ba0c9e800. 00010000:00000001:3.0:1773941420.800312:0:12100:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000010:3.1:1773941420.800872:0:417:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138ea00. 00010000:00000010:3.1:1773941420.800877:0:417:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec400. 00010000:00000001:0.0:1773941420.801499:0:12100:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00010000:00000001:0.0:1773941420.801501:0:12100:0:(ldlm_resource.c:1329:ldlm_namespace_free_post()) Process leaving 00010000:00000001:0.0:1773941420.801503:0:12100:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 00010000:00000001:0.0:1773941420.801505:0:12100:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 00010000:00000001:0.0:1773941420.801506:0:12100:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:0.0:1773941420.801508:0:12100:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00010000:00000010:0.0:1773941420.801510:0:12100:0:(ldlm_lib.c:642:client_obd_cleanup()) kfreed 'cli->cl_mod_tag_bitmap': 64 at ffff8b5b82c59340. 00010000:00000010:0.0:1773941420.801514:0:12100:0:(ldlm_lib.c:647:client_obd_cleanup()) kfreed 'cli->cl_read_io_latency_by_size': 3920 at ffff8b5b85e5c000. 00010000:00000010:0.0:1773941420.801517:0:12100:0:(ldlm_lib.c:651:client_obd_cleanup()) kfreed 'cli->cl_write_io_latency_by_size': 3920 at ffff8b5b85e5e000. 00010000:00000001:0.0:1773941420.801519:0:12100:0:(ldlm_lib.c:654:client_obd_cleanup()) Process leaving 00000008:00000001:0.0:1773941420.801521:0:12100:0:(osc_request.c:4167:osc_cleanup_common()) Process leaving (rc=0 : 0 : 0) 00000002:00000010:0.0:1773941420.801524:0:12100:0:(mdc_dev.c:1712:mdc_device_free()) kfreed 'osc': 104 at ffff8b5b90edb580. 00000020:00000010:0.0:1773941420.801528:0:12100:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b8809ce00. 00000020:00000010:0.0:1773941420.801534:0:12100:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc91e0. 00008000:00000010:0.0:1773941420.801537:0:12100:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b98a1e6d8. 00000020:00000010:0.0:1773941420.801540:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b8809ca00. 00000020:00000001:0.0:1773941420.801542:0:12100:0:(obd_class.h:647:obd_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941420.801544:0:12100:0:(genops.c:66:obd_device_free()) slab-freed '(obd)': 6584 at ffff8b5b855119b8. 00000020:00000001:0.0:1773941420.801551:0:12100:0:(obd_config.c:923:class_detach()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941420.801553:0:12100:0:(obd_config.c:1655:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1773941420.801555:0:12100:0:(obd_config.c:2523:class_manual_cleanup()) kfreed 'lcfg': 88 at ffff8b5ba0bc9a20. 00000020:00000001:0.0:1773941420.801558:0:12100:0:(obd_config.c:2524:class_manual_cleanup()) Process leaving (rc=0 : 0 : 0) 00000080:02000400:0.0:1773941420.801561:0:12100:0:(llite_lib.c:1617:ll_put_super()) Unmounted lustre-client 00000020:00000001:0.0:1773941420.802636:0:12100:0:(obd_config.c:1231:class_del_profile()) Process entered 00000020:01000000:0.0:1773941420.802638:0:12100:0:(obd_config.c:1233:class_del_profile()) Del profile lustre-client 00000020:00000001:0.0:1773941420.802639:0:12100:0:(obd_config.c:1149:class_get_profile_nolock()) Process entered 00000020:00000001:0.0:1773941420.802641:0:12100:0:(obd_config.c:1153:class_get_profile_nolock()) Process leaving (rc=18446615823885113600 : -128249824438016 : ffff8b5b82c59900) 00000020:00000010:0.0:1773941420.802647:0:12100:0:(obd_config.c:1270:class_put_profile()) kfreed 'lprof->lp_profile': 14 at ffff8b5b84f3fb90. 00000020:00000010:0.0:1773941420.802649:0:12100:0:(obd_config.c:1271:class_put_profile()) kfreed 'lprof->lp_dt': 14 at ffff8b5b84f3f4d0. 00000020:00000010:0.0:1773941420.802658:0:12100:0:(obd_config.c:1272:class_put_profile()) kfreed 'lprof->lp_md': 14 at ffff8b5b84f3fee0. 00000020:00000010:0.0:1773941420.802660:0:12100:0:(obd_config.c:1273:class_put_profile()) kfreed 'lprof': 48 at ffff8b5b82c59900. 00000020:00000001:0.0:1773941420.802663:0:12100:0:(obd_config.c:1248:class_del_profile()) Process leaving 00000080:00000001:0.0:1773941420.802664:0:12100:0:(llite_lib.c:266:ll_free_sbi()) Process entered 00000020:00000010:0.0:1773941420.802682:0:12100:0:(cl_page.c:1415:cl_cache_decref()) kfreed 'cache': 152 at ffff8b5b85339e40. 00000080:00000010:0.0:1773941420.802687:0:12100:0:(llite_lib.c:281:ll_free_sbi()) kfreed 'sbi->ll_foreign_symlink_prefix': 6 at ffff8b5b893196b0. 00000080:00000010:0.0:1773941420.802696:0:12100:0:(llite_lib.c:287:ll_free_sbi()) kfreed 'sbi->ll_foreign_symlink_upcall': 5 at ffff8b5b89319488. 00000080:00000010:0.0:1773941420.802759:0:12100:0:(llite_lib.c:310:ll_free_sbi()) kfreed 'sbi': 1376 at ffff8b5b98ac1000. 00000080:00000001:0.0:1773941420.802765:0:12100:0:(llite_lib.c:312:ll_free_sbi()) Process leaving 00000020:00000001:0.0:1773941420.802768:0:12100:0:(obd_mount.c:909:lustre_common_put_super()) Process entered 00000020:01000004:0.0:1773941420.802770:0:12100:0:(obd_mount.c:911:lustre_common_put_super()) dropping sb ffff8b5b885e5800 00000020:00000001:0.0:1773941420.802772:0:12100:0:(obd_mount.c:575:lustre_stop_mgc()) Process entered 00000020:00000001:0.0:1773941420.802775:0:12100:0:(obd_class.h:902:obd_disconnect()) Process entered 00010000:00000001:0.0:1773941420.802778:0:12100:0:(ldlm_lib.c:759:client_disconnect_export()) Process entered 00010000:00000040:0.0:1773941420.802780:0:12100:0:(ldlm_lib.c:772:client_disconnect_export()) disconnect MGC192.168.204.143@tcp - 1 00000100:00000001:0.0:1773941420.802784:0:12100:0:(pinger.c:388:ptlrpc_pinger_del_import()) Process entered 00000100:00080000:0.0:1773941420.802786:0:12100:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import f10576f4-3fb6-413e-88ca-8953d457ee86->MGS 00000020:00000001:0.0:1773941420.802788:0:12100:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941420.802790:0:12100:0:(genops.c:1242:class_import_put()) import ffff8b5b88954000 refcount=2 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941420.802793:0:12100:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941420.802795:0:12100:0:(pinger.c:401:ptlrpc_pinger_del_import()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941420.802798:0:12100:0:(ldlm_request.c:2367:ldlm_cli_cancel_unused()) Process entered 00000001:00000001:0.0:1773941420.802803:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941420.802805:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00010000:00000040:0.0:1773941420.802809:0:12100:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8b5b83962f00 count: 2 00010000:00000001:0.0:1773941420.802813:0:12100:0:(ldlm_request.c:2311:ldlm_cli_cancel_unused_resource()) Process entered 00010000:00000040:0.0:1773941420.802817:0:12100:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8b5b83962f00 count: 3 00010000:00000001:0.0:1773941420.802820:0:12100:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1773941420.802822:0:12100:0:(ldlm_request.c:2161:ldlm_cancel_resource_local()) Process entered 00010000:00000001:0.0:1773941420.802826:0:12100:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1773941420.802831:0:12100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1be2 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1008400000000 nid: local remote: 0xb574482a8fd85f74 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1773941420.802840:0:12100:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1773941420.802843:0:12100:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1be2 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1009400000000 nid: local remote: 0xb574482a8fd85f74 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1773941420.802857:0:12100:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:01000000:0.0:1773941420.802860:0:12100:0:(mgc_request.c:954:mgc_blocking_ast()) log lustre-s: disconnecting, won't requeue 10000000:00000001:0.0:1773941420.802861:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1773941420.802862:0:12100:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b839c6e00) refs 1 10000000:01000000:0.0:1773941420.802864:0:12100:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-sptlrpc 10000000:00000001:0.0:1773941420.802865:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941420.802866:0:12100:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941420.802866:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941420.802867:0:12100:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941420.802868:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941420.802868:0:12100:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941420.802869:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941420.802870:0:12100:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941420.802871:0:12100:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941420.802871:0:12100:0:(mgc_request.c:120:config_log_put()) Process leaving 02000000:08000000:0.0:1773941420.802875:0:12100:0:(sec_config.c:500:sptlrpc_conf_free()) free sptlrpc conf lustre 02000000:00000010:0.0:1773941420.802876:0:12100:0:(sec_config.c:504:sptlrpc_conf_free()) kfreed 'conf': 120 at ffff8b5b9020ab80. 00000020:00000040:0.0:1773941420.802879:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b885e2000 : new refcount 2 10000000:00000010:0.0:1773941420.802881:0:12100:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 287 at ffff8b5b839c6e00. 10000000:00000001:0.0:1773941420.802884:0:12100:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:0.0:1773941420.802885:0:12100:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941420.802887:0:12100:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1773941420.802889:0:12100:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1773941420.802891:0:12100:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1773941420.802892:0:12100:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941420.802894:0:12100:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941420.802897:0:12100:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1be2 from hash 00010000:00000001:0.0:1773941420.802898:0:12100:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1773941420.802899:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941420.802901:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941420.802902:0:12100:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1773941420.802903:0:12100:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1773941420.802904:0:12100:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:0.0:1773941420.802906:0:12100:0:(ldlm_request.c:1740:ldlm_cli_cancel_list_local()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1773941420.802908:0:12100:0:(ldlm_request.c:2211:ldlm_cancel_resource_local()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1773941420.802915:0:12100:0:(ldlm_request.c:2242:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1773941420.802918:0:12100:0:(ldlm_request.c:1396:ldlm_cli_cancel_req()) Process entered 00000100:00000010:0.0:1773941420.802924:0:12100:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b850a8700. 00000020:00000040:0.0:1773941420.802928:0:12100:0:(genops.c:1229:class_import_get()) import ffff8b5b88954000 refcount=3 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941420.802932:0:12100:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941420.802935:0:12100:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941420.802939:0:12100:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941420.802945:0:12100:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b84feea00. 00000100:00000001:0.0:1773941420.802951:0:12100:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941420.802954:0:12100:0:(ldlm_request.c:1348:ldlm_cancel_pack()) Process entered 00010000:00010000:0.0:1773941420.802960:0:12100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1be2 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85f74 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941420.802967:0:12100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:0.0:1773941420.802968:0:12100:0:(ldlm_request.c:1380:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941420.802972:0:12100:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941420.802974:0:12100:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941420.802979:0:12100:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b6ed80. 00000100:00000001:0.0:1773941420.802982:0:12100:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387468672 : -128249322082944 : ffff8b5ba0b6ed80) 00000100:00000001:0.0:1773941420.802984:0:12100:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941420.802986:0:12100:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615823923185408 : -128249786366208 : ffff8b5b850a8700) 00000100:00000001:0.0:1773941420.802990:0:12100:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941420.802994:0:12100:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.802996:0:12100:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941420.802998:0:12100:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941420.803007:0:12100:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b850a8700 x1860112241144192/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941420.803015:0:12100:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941420.803017:0:12100:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941420.803022:0:12100:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b850a8700 x1860112241144192/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941420.803029:0:12100:0:(client.c:1841:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1773941420.803031:0:12100:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941420.803033:0:12100:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941420.803042:0:12100:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b6ed80 going to sleep for 0 seconds 00000100:00000001:0.0:1773941420.803045:0:12100:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941420.803047:0:12100:0:(client.c:2089:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:0.0:1773941420.803051:0:12100:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941420.803054:0:12100:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941420.803056:0:12100:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941420.803061:0:12100:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b850a8700 x1860112241144192/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941420.803070:0:12100:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8700 pname:cluuid:pid:xid:nid:opc:job umount:f10576f4-3fb6-413e-88ca-8953d457ee86:12100:1860112241144192:192.168.204.143@tcp:103:umount.0 00000100:00000001:0.0:1773941420.803076:0:12100:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941420.803078:0:12100:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:0.0:1773941420.803081:0:12100:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941420.803083:0:12100:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941420.803087:0:12100:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b850a8700 x1860112241144192/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941420.803094:0:12100:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941420.803096:0:12100:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b6ed80. 00000100:00000001:0.0:1773941420.803100:0:12100:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941420.803101:0:12100:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00010000:0.0:1773941420.803104:0:12100:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00000100:00000001:0.0:1773941420.803106:0:12100:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941420.803109:0:12100:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b850a8700 x1860112241144192/t0(0) o103->MGC192.168.204.143@tcp@192.168.204.143@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941420.803117:0:12100:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1773941420.803118:0:12100:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941420.803120:0:12100:0:(genops.c:1242:class_import_put()) import ffff8b5b88954000 refcount=2 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941420.803122:0:12100:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941420.803125:0:12100:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b84feea00. 02000000:00000001:0.0:1773941420.803128:0:12100:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941420.803130:0:12100:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941420.803132:0:12100:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b850a8700. 00000100:00000001:0.0:1773941420.803135:0:12100:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941420.803142:0:12100:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1773941420.803144:0:12100:0:(ldlm_request.c:1501:ldlm_cli_cancel_req()) Process leaving 00010000:00010000:0.0:1773941420.803146:0:12100:0:(ldlm_request.c:2277:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00000001:0.0:1773941420.803148:0:12100:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1773941420.803151:0:12100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1be2 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5809400000020 nid: local remote: 0xb574482a8fd85f74 expref: -99 pid: 12017 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941420.803161:0:12100:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941420.803163:0:12100:0:(ldlm_request.c:2290:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941420.803165:0:12100:0:(ldlm_request.c:2328:ldlm_cli_cancel_unused_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:0.0:1773941420.803168:0:12100:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962f00 count: 0 00000001:00000001:0.0:1773941420.803175:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941420.803178:0:12100:0:(ldlm_request.c:2379:ldlm_cli_cancel_unused()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941420.803181:0:12100:0:(ldlm_resource.c:1268:ldlm_namespace_free_prior()) Process entered 00010000:00000001:0.0:1773941420.803183:0:12100:0:(ldlm_resource.c:1207:__ldlm_namespace_free()) Process entered 00000001:00000001:0.0:1773941420.803184:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941420.803186:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941420.803188:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941420.803191:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941420.803192:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941420.803194:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941420.803197:0:12100:0:(ldlm_resource.c:1251:__ldlm_namespace_free()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941420.803199:0:12100:0:(ldlm_resource.c:1293:ldlm_namespace_free_prior()) Process leaving 00000100:00000001:0.0:1773941420.803201:0:12100:0:(import.c:1924:ptlrpc_disconnect_import()) Process entered 00000100:00000001:0.0:1773941420.803205:0:12100:0:(import.c:1858:ptlrpc_disconnect_import_async()) Process entered 00000100:00000001:0.0:1773941420.803207:0:12100:0:(import.c:1750:ptlrpc_disconnect_prep_req()) Process entered 00000100:00000010:0.0:1773941420.803210:0:12100:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b850a8700. 00000020:00000040:0.0:1773941420.803213:0:12100:0:(genops.c:1229:class_import_get()) import ffff8b5b88954000 refcount=3 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941420.803216:0:12100:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941420.803218:0:12100:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941420.803220:0:12100:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941420.803224:0:12100:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b99101500. 00000100:00000001:0.0:1773941420.803227:0:12100:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.803230:0:12100:0:(import.c:1788:ptlrpc_disconnect_prep_req()) Process leaving (rc=18446615823923185408 : -128249786366208 : ffff8b5b850a8700) 00000100:00080000:0.0:1773941420.803233:0:12100:0:(import.c:65:import_set_state_nolock()) ffff8b5b88954000 MGS: changing import state from FULL to CONNECTING 00000100:00000001:0.0:1773941420.803236:0:12100:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941420.803243:0:12100:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941420.803249:0:12100:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000373a969d] to pc [ptlrpcd_rcv+-1] req@ffff8b5b850a8700 x1860112241144320/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941420.803278:0:12100:0:(import.c:1904:ptlrpc_disconnect_import_async()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.803293:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941420.803297:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941420.803298:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941420.803304:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b850a8700 x1860112241144320/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941420.803311:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941420.803312:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941420.803315:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941420.803317:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941420.803319:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941420.803321:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f10576f4-3fb6-413e-88ca-8953d457ee86:2351:1860112241144320:192.168.204.143@tcp:251:umount.0 00000100:00000001:2.0:1773941420.803325:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941420.803328:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941420.803331:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941420.803333:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941420.803338:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b89316200. 02000000:00000001:2.0:1773941420.803340:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941420.803348:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941420.803352:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9ec8. 00000400:00000010:2.0:1773941420.803359:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34770. 00000100:00000200:2.0:1773941420.803366:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241144320, portal 25 00000100:00000001:2.0:1773941420.803368:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941420.803370:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615823923185408 : -128249786366208 : ffff8b5b850a8700) 00000100:00000040:2.0:1773941420.803377:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b850a8700 x1860112241144320/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941420.803385:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941420.803395:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941420.803398:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34ee0. 00000100:00000200:2.0:1773941420.803401:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241144320, offset 0 00000400:00000200:2.0:1773941420.803406:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941420.803413:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941420.803419:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941420.803423:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941420.803427:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941420.803430:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941420.803432:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941420.803436:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [105:105:256:4294967295] 192.168.204.143@tcp LPNI seq info [105:105:8:4294967295] 00000400:00000200:2.0:1773941420.803439:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941420.803449:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941420.803454:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941420.803458:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0c00. 00000800:00000200:2.0:1773941420.803464:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941420.803470:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941420.803474:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0c00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1773941420.803484:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941420.803488:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941420.803490:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294861166 00000100:00000001:2.0:1773941420.803495:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294861147 00000100:00000001:2.0:1773941420.803500:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294861166 00000100:00000001:2.0:1773941420.803503:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294861147 00000100:00000001:2.0:1773941420.803506:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941420.803508:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.803510:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.803513:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941420.803518:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941420.803520:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941420.803521:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941420.803527:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941420.803528:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.803530:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941420.803599:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941420.803602:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0c00. 00000400:00000200:2.0:1773941420.803606:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941420.803610:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941420.803614:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34ee0 00000400:00000010:2.0:1773941420.803616:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34ee0. 00000100:00000001:2.0:1773941420.803618:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941420.803622:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b850a8700 x1860112241144320/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941430 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941420.803628:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941420.803631:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b850a8700 x1860112241144320/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941430 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941420.803635:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.803636:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941420.805277:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.805287:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941420.805289:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.805292:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.805298:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.805303:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941420.805310:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.805314:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.805318:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.805322:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321603200 00000400:00000200:3.0:1773941420.805327:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941420.805332:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x8a5 [1] + 224 00000800:00000001:3.0:1773941420.805339:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.805345:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.805348:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941420.805352:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941420.805357:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941420.805363:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b850a8700 x1860112241144320/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941420.805373:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b850a8700 x1860112241144320/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.805387:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941420.805397:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.805401:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941420.805422:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941420.805425:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941420.805429:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.805431:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.805434:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9ec8. 00000400:00000200:3.0:1773941420.805438:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34770 00000400:00000010:3.0:1773941420.805440:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34770. 00000100:00000001:3.0:1773941420.805443:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941420.805447:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b850a8700 x1860112241144320/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941420.805464:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b850a8700 x1860112241144320/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.805470:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941420.805472:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941420.805475:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941420.805478:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941420.805480:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941420.805482:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941420.805484:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941420.805489:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b88954458 time=6 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941420.805492:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b88954410 time=6 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941420.805494:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941420.805495:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.805497:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941420.805499:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.805501:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941420.805504:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b850a8700 x1860112241144320/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.805509:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941420.805510:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941420.805512:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941420.805514:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b88954000 MGS: changing import state from CONNECTING to CLOSED 00000100:00000001:3.0:1773941420.805516:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941420.805518:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941420.805520:0:2351:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941420.805521:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.805522:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941420.805522:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941420.805523:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941420.805524:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.805525:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000040:3.0:1773941420.805535:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b850a8700 x1860112241144320/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941420.805550:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f10576f4-3fb6-413e-88ca-8953d457ee86:2351:1860112241144320:192.168.204.143@tcp:251:umount.0 00000100:00000001:3.0:1773941420.805553:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941420.805555:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941420.805558:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b850a8700 x1860112241144320/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941430 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941420.805562:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941420.805564:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941420.805566:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b89316200. 02000000:00000001:3.0:1773941420.805568:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941420.805569:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941420.805571:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b88954000 refcount=2 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941420.805572:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941420.805573:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b99101500. 02000000:00000001:3.0:1773941420.805576:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941420.805577:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941420.805579:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b850a8700. 00000100:00000001:3.0:1773941420.805581:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941420.805582:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941420.805583:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941420.805588:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941420.805589:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.805590:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941420.805591:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.805604:0:12100:0:(import.c:1967:ptlrpc_disconnect_import()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.805607:0:12100:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941420.805608:0:12100:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941420.805609:0:12100:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:3.0:1773941420.805611:0:12100:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941420.805612:0:12100:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941420.805613:0:12100:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941420.805614:0:12100:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941420.805616:0:12100:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.805617:0:12100:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00100000:3.0:1773941420.805618:0:12100:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941420.805624:0:12100:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941420.805625:0:12100:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941420.805626:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.805628:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.805630:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.805632:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.805633:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.805634:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941420.805635:0:12100:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.805636:0:12100:0:(obd_class.h:1226:obd_import_event()) Process leaving 00010000:00000001:3.0:1773941420.805639:0:12100:0:(ldlm_lib.c:818:client_disconnect_export()) Process leaving 00000020:00000001:3.0:1773941420.805640:0:12100:0:(genops.c:1462:class_disconnect()) Process entered 00000020:00000080:3.0:1773941420.805642:0:12100:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0xa33a5be14eae1bdb 00000020:00000040:3.0:1773941420.805645:0:12100:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b885e6800 with handle 0xa33a5be14eae1bdb from hash 00000020:00000001:3.0:1773941420.805649:0:12100:0:(genops.c:1934:obd_stale_export_put()) Process entered 00000020:00000040:3.0:1773941420.805650:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b885e6800 : new refcount 1 00000020:00000001:3.0:1773941420.805652:0:12100:0:(genops.c:1959:obd_stale_export_put()) Process leaving 00000020:00000040:3.0:1773941420.805653:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b885e6800 : new refcount 0 00000020:00000080:3.0:1773941420.805654:0:12100:0:(genops.c:904:class_export_put()) final put ffff8b5b885e6800/f10576f4-3fb6-413e-88ca-8953d457ee86 00000020:00000001:3.0:1773941420.805656:0:12100:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.805664:0:12100:0:(genops.c:1493:class_disconnect()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.805666:0:12100:0:(ldlm_lib.c:831:client_disconnect_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.805668:0:12100:0:(obd_class.h:914:obd_disconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.805670:0:12100:0:(obd_config.c:2489:class_manual_cleanup()) Process entered 00000020:01000000:3.0:1773941420.805671:0:12100:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of MGC192.168.204.143@tcp (flags='') 00000020:00000001:3.0:1773941420.805678:0:37:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:3.0:1773941420.805680:0:37:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b885e6800/f10576f4-3fb6-413e-88ca-8953d457ee86 for MGC192.168.204.143@tcp 00000020:00000040:3.0:1773941420.805683:0:37:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000020:00000001:3.0:1773941420.805685:0:37:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:3.0:1773941420.805687:0:37:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941420.805689:0:37:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.143@tcp (ffff8b5b85514d28) now 5 - export 00000020:00000010:3.0:1773941420.805691:0:37:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff8b5b885e6800. 00000020:00000001:3.0:1773941420.805693:0:37:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000010:3.0:1773941420.805698:0:12100:0:(obd_config.c:2506:class_manual_cleanup()) kmalloced '(lcfg)': 72 at ffff8b5ba0bc9f00. 00000020:00000080:3.0:1773941420.805700:0:12100:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000001:3.0:1773941420.805730:0:12100:0:(obd_config.c:936:class_cleanup()) Process entered 00000020:00000080:3.0:1773941420.805732:0:12100:0:(obd_config.c:991:class_cleanup()) MGC192.168.204.143@tcp: forcing exports to disconnect: 0/2 00000020:00000001:3.0:1773941420.805735:0:12100:0:(genops.c:1562:class_disconnect_exports()) Process entered 00000020:00080000:3.0:1773941420.805737:0:12100:0:(genops.c:1577:class_disconnect_exports()) OBD device 0 (ffff8b5b85514d28) has no exports 00000020:00000001:3.0:1773941420.805738:0:12100:0:(genops.c:1578:class_disconnect_exports()) Process leaving 00000020:00000001:3.0:1773941420.805740:0:12100:0:(obd_class.h:604:obd_precleanup()) Process entered 00000020:00000010:3.0:1773941420.805743:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b8809c800. 00000020:00000010:3.0:1773941420.805747:0:12100:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b8809d400. 00000040:00000010:3.0:1773941420.805751:0:12100:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b8809da00. 00000020:00000010:3.0:1773941420.805755:0:12100:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b98ac3000. 00000020:00000010:3.0:1773941420.805758:0:12100:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b8809c600. 00040000:00000010:3.0:1773941420.805762:0:12100:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98a69900. 00040000:00000010:3.0:1773941420.805766:0:12100:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b8809c200. 00000020:00000010:3.0:1773941420.805770:0:12100:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068100. 10000000:00000001:3.0:1773941420.805773:0:12100:0:(mgc_request.c:792:mgc_device_fini()) Process entered 10000000:01000000:3.0:1773941420.805783:0:12031:0:(mgc_request.c:708:mgc_requeue_thread()) Ending requeue thread 10000000:00000001:3.0:1773941420.805785:0:12031:0:(mgc_request.c:709:mgc_requeue_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941420.805817:0:12100:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 10000000:01000000:3.0:1773941420.805819:0:12100:0:(obd_class.h:663:obd_cleanup_client_import()) MGC192.168.204.143@tcp: client import never connected 00000100:00000001:3.0:1773941420.805821:0:12100:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941420.805822:0:12100:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941420.805824:0:12100:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:3.0:1773941420.805825:0:12100:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941420.805826:0:12100:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941420.805828:0:12100:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941420.805829:0:12100:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941420.805830:0:12100:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.805831:0:12100:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00100000:3.0:1773941420.805832:0:12100:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941420.805834:0:12100:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941420.805834:0:12100:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941420.805835:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.805836:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.805837:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.805839:0:12100:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941420.805839:0:12100:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941420.805841:0:12100:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941420.805847:0:12100:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.805848:0:12100:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:3.0:1773941420.805850:0:12100:0:(genops.c:1229:class_import_get()) import ffff8b5b88954000 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941420.805852:0:12100:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941420.805853:0:12100:0:(genops.c:1242:class_import_put()) import ffff8b5b88954000 refcount=2 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941420.805855:0:12100:0:(genops.c:1249:class_import_put()) Process leaving 02000000:08000000:3.0:1773941420.805858:0:12100:0:(sec.c:1366:sec_cop_destroy_sec()) sec.null@ffffffffc1067ea0: being destroyed 00000020:00000001:3.0:1773941420.805860:0:12100:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941420.805861:0:12100:0:(genops.c:1242:class_import_put()) import ffff8b5b88954000 refcount=1 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941420.805862:0:12100:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000001:3.0:1773941420.805863:0:12100:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 10000000:00000001:3.0:1773941420.805864:0:12100:0:(mgc_request.c:774:mgc_llog_fini()) Process entered 10000000:00000040:3.0:1773941420.805866:0:12100:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5b885de3c0 : new refcount 2 00000040:00000001:3.0:1773941420.805868:0:12100:0:(llog_obd.c:93:llog_cleanup()) Process entered 00000040:00000040:3.0:1773941420.805869:0:12100:0:(lustre_log.h:400:llog_ctxt_put()) PUTting ctxt ffff8b5b885de3c0 : new refcount 1 00000020:00000040:3.0:1773941420.805871:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b885e2000 : new refcount 1 00000020:00000001:3.0:1773941420.805872:0:12100:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941420.805873:0:12100:0:(genops.c:1242:class_import_put()) import ffff8b5b88954000 refcount=0 obd=MGC192.168.204.143@tcp 00000020:00000040:3.0:1773941420.805875:0:12100:0:(genops.c:1245:class_import_put()) final put import ffff8b5b88954000 00000020:00000001:3.0:1773941420.805881:0:12100:0:(genops.c:1249:class_import_put()) Process leaving 00000040:00000010:3.0:1773941420.805882:0:12100:0:(llog_obd.c:46:llog_ctxt_destroy()) kfreed 'ctxt': 168 at ffff8b5b885de3c0. 00000020:00000001:3.0:1773941420.805890:0:37:0:(genops.c:1200:obd_zombie_import_free()) Process entered 00000020:00000080:3.0:1773941420.805891:0:37:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff8b5b88954000 for MGC192.168.204.143@tcp 00000020:00000040:3.0:1773941420.805894:0:37:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 1 to 192.168.204.143@tcp 00000020:00000040:3.0:1773941420.805896:0:37:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 0 to 192.168.204.143@tcp 00000020:00000010:3.0:1773941420.805898:0:37:0:(genops.c:1213:obd_zombie_import_free()) kfreed 'imp_conn': 88 at ffff8b5b84cfb4e0. 00000020:00000040:3.0:1773941420.805900:0:37:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.143@tcp (ffff8b5b85514d28) now 4 - import 00000020:00000010:3.0:1773941420.805902:0:37:0:(genops.c:1220:obd_zombie_import_free()) kfreed 'imp': 1712 at ffff8b5b88954000. 00000020:00000001:3.0:1773941420.805905:0:37:0:(genops.c:1221:obd_zombie_import_free()) Process leaving 00000040:00000001:3.0:1773941420.805909:0:12100:0:(llog_obd.c:122:llog_cleanup()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941420.805911:0:12100:0:(mgc_request.c:780:mgc_llog_fini()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941420.805913:0:12100:0:(mgc_request.c:816:mgc_device_fini()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.805916:0:12100:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068100. 00000020:00000010:3.0:1773941420.805924:0:12100:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b8809d400. 00000040:00000010:3.0:1773941420.805927:0:12100:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b8809da00. 00000020:00000010:3.0:1773941420.805934:0:12100:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b98ac3000. 00000020:00000010:3.0:1773941420.805938:0:12100:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b8809c600. 00040000:00000010:3.0:1773941420.805941:0:12100:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98a69900. 00040000:00000010:3.0:1773941420.805944:0:12100:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b8809c200. 00000020:00000010:3.0:1773941420.805946:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b8809c800. 00000020:00000001:3.0:1773941420.805949:0:12100:0:(obd_class.h:623:obd_precleanup()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941420.805961:0:12100:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:3.0:1773941420.805965:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5b902a1e00. 00000001:00000010:3.0:1773941420.805966:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5b902a1800. 00000001:00000010:3.0:1773941420.805967:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5b902a1200. 00000001:00000010:3.0:1773941420.805969:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5b902a0c00. 00000001:00000010:3.0:1773941420.805970:0:12100:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 32 at ffff8b5b88570c80. 00000001:00000010:3.0:1773941420.805972:0:12100:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 232 at ffff8b5ba0c22900. 00000001:00000001:3.0:1773941420.805974:0:12100:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00000020:00000040:3.0:1773941420.805975:0:12100:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.143@tcp (ffff8b5b85514d28) now 3 - setup 00000020:00000001:3.0:1773941420.805977:0:12100:0:(obd_config.c:1021:class_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.805978:0:12100:0:(obd_config.c:1659:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000080:3.0:1773941420.805979:0:12100:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000001:3.0:1773941420.805981:0:12100:0:(obd_config.c:899:class_detach()) Process entered 00000020:00000040:3.0:1773941420.805984:0:12100:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.143@tcp (ffff8b5b85514d28) now 2 - obd_device_list 00000020:00000080:3.0:1773941420.805987:0:12100:0:(obd_config.c:919:class_detach()) detach on obd MGC192.168.204.143@tcp (uuid f10576f4-3fb6-413e-88ca-8953d457ee86) 00000020:00000040:3.0:1773941420.805988:0:12100:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.143@tcp (ffff8b5b85514d28) now 1 - newdev 00000020:00000040:3.0:1773941420.805990:0:12100:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b885e2000 with handle 0xa33a5be14eae1bd4 from hash 00000020:00000040:3.0:1773941420.805991:0:12100:0:(genops.c:898:class_export_put()) PUTting export ffff8b5b885e2000 : new refcount 0 00000020:00000080:3.0:1773941420.805993:0:12100:0:(genops.c:904:class_export_put()) final put ffff8b5b885e2000/f10576f4-3fb6-413e-88ca-8953d457ee86 00000020:00000001:3.0:1773941420.805994:0:12100:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.805995:0:12100:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:3.0:1773941420.805996:0:12100:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5b885e2000/f10576f4-3fb6-413e-88ca-8953d457ee86 for MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941420.805997:0:12100:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:3.0:1773941420.805998:0:12100:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.806000:0:12100:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff8b5b885e2000. 00000020:00000001:3.0:1773941420.806003:0:12100:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000040:3.0:1773941420.806004:0:12100:0:(genops.c:437:class_free_dev()) Release obd device MGC192.168.204.143@tcp obd_type name = mgc 00000020:01000000:3.0:1773941420.806010:0:12100:0:(genops.c:440:class_free_dev()) finishing cleanup of obd MGC192.168.204.143@tcp (f10576f4-3fb6-413e-88ca-8953d457ee86) 00000020:00000001:3.0:1773941420.806012:0:12100:0:(obd_class.h:633:obd_cleanup()) Process entered 00000020:00000010:3.0:1773941420.806014:0:12100:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b8809c800. 00000020:00000010:3.0:1773941420.806017:0:12100:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b8809c200. 00000040:00000010:3.0:1773941420.806020:0:12100:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b8809c600. 00000020:00000010:3.0:1773941420.806023:0:12100:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b98ac3000. 00000020:00000010:3.0:1773941420.806026:0:12100:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b8809da00. 00040000:00000010:3.0:1773941420.806029:0:12100:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98a69900. 00040000:00000010:3.0:1773941420.806032:0:12100:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b8809d400. 10000000:00000001:3.0:1773941420.806035:0:12100:0:(mgc_request.c:824:mgc_device_free()) Process entered 00000020:00000001:3.0:1773941420.806036:0:12100:0:(obd_config.c:1281:class_del_profiles()) Process entered 00000020:00000001:3.0:1773941420.806038:0:12100:0:(obd_config.c:1294:class_del_profiles()) Process leaving 00010000:00000001:3.0:1773941420.806228:0:12100:0:(ldlm_lib.c:631:client_obd_cleanup()) Process entered 00010000:00000001:3.0:1773941420.806230:0:12100:0:(ldlm_resource.c:1304:ldlm_namespace_free_post()) Process entered 00010000:00000001:3.0:1773941420.806233:0:12100:0:(ldlm_pool.c:931:ldlm_pool_fini()) Process entered 00000020:00000010:3.0:1773941420.806284:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 448 at ffff8b5ba0f90800. 00000020:00000010:3.0:1773941420.806287:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 352 at ffff8b5ba0f91600. 00000020:00000010:3.0:1773941420.806290:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5ba0c28800. 00010000:00000001:3.0:1773941420.806337:0:12100:0:(ldlm_pool.c:934:ldlm_pool_fini()) Process leaving 00000020:00000010:3.0:1773941420.806360:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff8b5b8848f400. 00000020:00000010:3.0:1773941420.806363:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff8b5b90edb800. 00000020:00000010:3.0:1773941420.806366:0:12100:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff8b5b9020a800. 00000020:00000010:3.0:1773941420.806369:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 96 at ffff8b5b84cfb960. 00000020:00000010:3.0:1773941420.806371:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5ba0c2bc00. 00000001:00000001:2.0:1773941420.806819:0:12100:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:2.0:1773941420.806824:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 304 at ffff8b5ba0f90600. 00000001:00000010:2.0:1773941420.806828:0:12100:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 304 at ffff8b5ba0f90200. 00000001:00000010:2.0:1773941420.806832:0:12100:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 16 at ffff8b5b84f3f310. 00000001:00000010:2.0:1773941420.806834:0:12100:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 280 at ffff8b5ba0f90000. 00000001:00000001:2.0:1773941420.806836:0:12100:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00010000:00000010:2.0:1773941420.806838:0:12100:0:(ldlm_resource.c:1327:ldlm_namespace_free_post()) kfreed 'ns': 848 at ffff8b5ba0c29400. 00010000:00000001:2.0:1773941420.806841:0:12100:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:2.0:1773941420.806843:0:12100:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00010000:00000001:2.0:1773941420.806844:0:12100:0:(ldlm_resource.c:1329:ldlm_namespace_free_post()) Process leaving 00010000:00000001:2.0:1773941420.806846:0:12100:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 00010000:00000001:2.0:1773941420.806848:0:12100:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 00010000:00000001:2.0:1773941420.806849:0:12100:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:2.0:1773941420.806851:0:12100:0:(ldlm_lockd.c:3527:ldlm_cleanup()) Process entered 00000001:00000010:2.0:1773941420.806863:0:12100:0:(shrinker_debug.c:208:ll_shrinker_free()) kfreed 's': 88 at ffff8b5b84cfb720. 00000001:00000010:2.0:1773941420.806866:0:12100:0:(shrinker_debug.c:208:ll_shrinker_free()) kfreed 's': 88 at ffff8b5b84cfbea0. 00000020:00000001:0.0:1773941420.806893:0:12029:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941420.806897:0:12029:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941420.806900:0:12029:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-1 blwis in pool) 00010000:00000001:0.0:1773941420.806903:0:12029:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:0.0:1773941420.806904:0:12029:0:(ldlm_lockd.c:2878:ldlm_bl_thread_blwi()) Process leaving (rc=2 : 2 : 2) 00000020:00000010:0.0:1773941420.806912:0:12029:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068280. 00000020:00000010:0.0:1773941420.806919:0:12029:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b84d8a800. 00000020:00000010:0.0:1773941420.806923:0:12029:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b9066e000. 00000020:00000010:0.0:1773941420.806927:0:12029:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b8849e4a0. 00000020:00000010:0.0:1773941420.806931:0:12029:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b84d8aa00. 80000000:00000010:0.0:1773941420.806934:0:12029:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9360. 40000000:00000010:0.0:1773941420.806938:0:12029:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b85068a80. 00100000:00000010:0.0:1773941420.806942:0:12029:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b9026d000. 00040000:00000010:0.0:1773941420.806944:0:12029:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b884beb00. 00040000:00000010:0.0:1773941420.806947:0:12029:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b84d8ba00. 00002000:00000010:0.0:1773941420.806951:0:12029:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0b91c00. 00000020:00000010:0.0:1773941420.806954:0:12029:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84d8ac00. 00010000:00000010:0.0:1773941420.806957:0:12029:0:(ldlm_lockd.c:3018:ldlm_bl_thread_main()) kfreed 'env': 56 at ffff8b5b85068d00. 00010000:00000001:0.0:1773941420.806960:0:12029:0:(ldlm_lockd.c:3019:ldlm_bl_thread_main()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.807743:0:12028:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941420.807746:0:12028:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941420.807748:0:12028:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-2 blwis in pool) 00010000:00000001:3.0:1773941420.807750:0:12028:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:3.0:1773941420.807751:0:12028:0:(ldlm_lockd.c:2878:ldlm_bl_thread_blwi()) Process leaving (rc=2 : 2 : 2) 00000020:00000010:3.0:1773941420.807760:0:12028:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b850680c0. 00000020:00000010:3.0:1773941420.807767:0:12028:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b839e3a00. 00000020:00000010:3.0:1773941420.807770:0:12028:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b9066b800. 00000020:00000010:3.0:1773941420.807773:0:12028:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b8849cb78. 00000020:00000010:3.0:1773941420.807776:0:12028:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b91e54c00. 80000000:00000010:3.0:1773941420.807784:0:12028:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9000. 40000000:00000010:3.0:1773941420.807786:0:12028:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b85068780. 00100000:00000010:3.0:1773941420.807788:0:12028:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b9066b000. 00040000:00000010:3.0:1773941420.807790:0:12028:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b884be900. 00040000:00000010:3.0:1773941420.807792:0:12028:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b91e55200. 00002000:00000010:3.0:1773941420.807795:0:12028:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0b92400. 00000020:00000010:3.0:1773941420.807798:0:12028:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b839e3000. 00010000:00000010:3.0:1773941420.807799:0:12028:0:(ldlm_lockd.c:3018:ldlm_bl_thread_main()) kfreed 'env': 56 at ffff8b5b85068d40. 00010000:00000001:3.0:1773941420.807801:0:12028:0:(ldlm_lockd.c:3019:ldlm_bl_thread_main()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941420.807962:0:12105:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941420.807964:0:12105:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941420.807966:0:12105:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-3 blwis in pool) 00010000:00000001:0.0:1773941420.807968:0:12105:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:0.0:1773941420.807969:0:12105:0:(ldlm_lockd.c:2878:ldlm_bl_thread_blwi()) Process leaving (rc=2 : 2 : 2) 00000020:00000010:0.0:1773941420.807979:0:12105:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068e00. 00000020:00000010:0.0:1773941420.807984:0:12105:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b91df5800. 00000020:00000010:0.0:1773941420.807987:0:12105:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b98ac6800. 00000020:00000010:0.0:1773941420.807990:0:12105:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b8849a840. 00000020:00000010:0.0:1773941420.807993:0:12105:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b91df5200. 80000000:00000010:0.0:1773941420.807996:0:12105:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc93c0. 40000000:00000010:0.0:1773941420.808003:0:12105:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b850682c0. 00100000:00000010:0.0:1773941420.808005:0:12105:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b98ac5000. 00040000:00000010:0.0:1773941420.808007:0:12105:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b91e1e300. 00040000:00000010:0.0:1773941420.808009:0:12105:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b91df5e00. 00002000:00000010:0.0:1773941420.808012:0:12105:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5b90261800. 00000020:00000010:0.0:1773941420.808016:0:12105:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b91df5a00. 00000020:00000001:3.0:1773941420.808019:0:12106:0:(genops.c:1914:obd_stale_export_get()) Process entered 00010000:00000010:0.0:1773941420.808020:0:12105:0:(ldlm_lockd.c:3018:ldlm_bl_thread_main()) kfreed 'env': 56 at ffff8b5b85068b80. 00000020:00000001:3.0:1773941420.808022:0:12106:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941420.808025:0:12105:0:(ldlm_lockd.c:3019:ldlm_bl_thread_main()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941420.808026:0:12106:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-4 blwis in pool) 00010000:00000001:3.0:1773941420.808030:0:12106:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:3.0:1773941420.808031:0:12106:0:(ldlm_lockd.c:2878:ldlm_bl_thread_blwi()) Process leaving (rc=2 : 2 : 2) 00000020:00000010:3.0:1773941420.808039:0:12106:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5ba039c940. 00000020:00000010:3.0:1773941420.808043:0:12106:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b88909a00. 00000020:00000010:3.0:1773941420.808048:0:12106:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5ba0b0c000. 00010000:00000010:2.0:1773941420.808048:0:12100:0:(ldlm_lockd.c:3556:ldlm_cleanup()) kfreed 'blp': 168 at ffff8b5b885de480. 00000020:00000010:3.0:1773941420.808056:0:12106:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b9829c670. 00000100:00000001:2.0:1773941420.808056:0:12100:0:(service.c:3841:ptlrpc_unregister_service()) Process entered 00000100:00000200:2.0:1773941420.808060:0:12100:0:(service.c:3843:ptlrpc_unregister_service()) ldlm_cbd: tearing down 00000020:00000010:3.0:1773941420.808062:0:12106:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b88908000. 80000000:00000010:3.0:1773941420.808064:0:12106:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b85ef6180. 40000000:00000010:3.0:1773941420.808070:0:12106:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5ba039c680. 00000100:00000001:2.0:1773941420.808072:0:12100:0:(service.c:3384:ptlrpc_stop_all_threads()) Process entered 00100000:00000010:3.0:1773941420.808073:0:12106:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5ba0b0e000. 00000100:00000001:2.0:1773941420.808075:0:12100:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 00040000:00000010:3.0:1773941420.808077:0:12106:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b99101300. 00000100:00000040:2.0:1773941420.808078:0:12100:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_cbd 00040000:00000010:3.0:1773941420.808080:0:12106:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b84fef200. 00002000:00000010:3.0:1773941420.808083:0:12106:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0bf3c00. 00000100:00000040:2.0:1773941420.808083:0:12100:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #1 00000020:00000010:3.0:1773941420.808087:0:12106:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b88909400. 00000100:00000040:2.0:1773941420.808088:0:12100:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #0 00010000:00000010:3.0:1773941420.808091:0:12106:0:(ldlm_lockd.c:3018:ldlm_bl_thread_main()) kfreed 'env': 56 at ffff8b5ba039c5c0. 00010000:00000001:3.0:1773941420.808094:0:12106:0:(ldlm_lockd.c:3019:ldlm_bl_thread_main()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941420.808108:0:12100:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cb #1 00000020:00000010:1.0:1773941420.808115:0:12018:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b862a5c00. 00000040:00000010:1.0:1773941420.808122:0:12018:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b862a5600. 00000020:00000010:1.0:1773941420.808126:0:12018:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e5000. 00000020:00000010:1.0:1773941420.808130:0:12018:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b98a05f98. 00000020:00000010:1.0:1773941420.808134:0:12018:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b862a4800. 80000000:00000010:1.0:1773941420.808137:0:12018:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b84cfbf00. 40000000:00000010:1.0:1773941420.808140:0:12018:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b82c595c0. 00100000:00000010:1.0:1773941420.808143:0:12018:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b885e3800. 00040000:00000010:1.0:1773941420.808147:0:12018:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5ba0c22b00. 00040000:00000010:1.0:1773941420.808152:0:12018:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b885e0000. 00040000:00000010:1.0:1773941420.808155:0:12018:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b862a5000. 00000004:00000010:1.0:1773941420.808159:0:12018:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b885e3000. 00000004:00000010:1.0:1773941420.808164:0:12018:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5b90ee1000. 00000004:00000010:1.0:1773941420.808168:0:12018:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b885e1800. 00002000:00000010:1.0:1773941420.808171:0:12018:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5b8800f000. 00000004:00000010:1.0:1773941420.808174:0:12018:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5b8801b800. 00000020:00000010:1.0:1773941420.808178:0:12018:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b88c95800. 00000020:00000010:1.0:1773941420.808181:0:12018:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b82c59880. 00000100:00000010:1.0:1773941420.808187:0:12018:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5b82c59d40. 00000100:00100000:1.0:1773941420.808190:0:12018:0:(service.c:3162:ptlrpc_main()) ldlm_cb00_000: service thread [ffff8b5b88c94600:12018] 0 exiting: rc = 1 00000020:00000010:0.0:1773941420.808238:0:12019:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab9c00. 00000040:00000010:0.0:1773941420.808244:0:12019:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab9600. 00000020:00000010:0.0:1773941420.808246:0:12019:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90db5000. 00000020:00000010:0.0:1773941420.808249:0:12019:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b9829eeb0. 00000020:00000010:0.0:1773941420.808253:0:12019:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab9a00. 80000000:00000010:0.0:1773941420.808257:0:12019:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b85ef6240. 40000000:00000010:0.0:1773941420.808260:0:12019:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5ba039c640. 00100000:00000010:0.0:1773941420.808263:0:12019:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b90db4000. 00040000:00000010:0.0:1773941420.808279:0:12019:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b99101700. 00040000:00000010:0.0:1773941420.808282:0:12019:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b90db6000. 00040000:00000010:0.0:1773941420.808285:0:12019:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab8800. 00000004:00000010:0.0:1773941420.808287:0:12019:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b90db3800. 00000004:00000010:0.0:1773941420.808290:0:12019:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5b98aca000. 00000004:00000010:0.0:1773941420.808295:0:12019:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b90db6800. 00002000:00000010:0.0:1773941420.808297:0:12019:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0bf2400. 00000004:00000010:0.0:1773941420.808299:0:12019:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5ba0bf2800. 00000020:00000010:0.0:1773941420.808302:0:12019:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab8400. 00000020:00000010:0.0:1773941420.808305:0:12019:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5ba039cb00. 00000100:00000010:0.0:1773941420.808308:0:12019:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5ba039cf00. 00000100:00100000:0.0:1773941420.808310:0:12019:0:(service.c:3162:ptlrpc_main()) ldlm_cb00_001: service thread [ffff8b5b885b0000:12019] 1 exiting: rc = 0 00000100:00000010:3.0:1773941420.808472:0:12100:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b88c94600. 00000100:00000010:3.0:1773941420.808476:0:12100:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b885b0000. 00000100:00000001:3.0:1773941420.808479:0:12100:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:3.0:1773941420.808481:0:12100:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 00000100:00000040:3.0:1773941420.808482:0:12100:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_cbd 00000100:00000040:3.0:1773941420.808484:0:12100:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #1 00000100:00000040:3.0:1773941420.808486:0:12100:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #0 00000100:00000040:3.0:1773941420.808500:0:12100:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cb #1 00000020:00000010:3.0:1773941420.808507:0:12020:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b885b0e00. 00000040:00000010:3.0:1773941420.808510:0:12020:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b885b1200. 00000020:00000010:3.0:1773941420.808511:0:12020:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d97800. 00000020:00000010:3.0:1773941420.808514:0:12020:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b88498000. 00000020:00000010:3.0:1773941420.808516:0:12020:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b885b1a00. 80000000:00000010:3.0:1773941420.808518:0:12020:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 40000000:00000010:3.0:1773941420.808520:0:12020:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b85068800. 00100000:00000010:3.0:1773941420.808522:0:12020:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b90d91000. 00040000:00000010:3.0:1773941420.808523:0:12020:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b884bef00. 00040000:00000010:3.0:1773941420.808525:0:12020:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b90d92800. 00040000:00000010:3.0:1773941420.808527:0:12020:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b885b1c00. 00000004:00000010:3.0:1773941420.808528:0:12020:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b90d93000. 00000004:00000010:3.0:1773941420.808530:0:12020:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5ba0b8e000. 00000004:00000010:3.0:1773941420.808533:0:12020:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b90d95800. 00002000:00000010:3.0:1773941420.808535:0:12020:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0b93800. 00000004:00000010:3.0:1773941420.808537:0:12020:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5ba0b92000. 00000020:00000010:3.0:1773941420.808539:0:12020:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b885b1400. 00000020:00000010:3.0:1773941420.808541:0:12020:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068cc0. 00000100:00000010:3.0:1773941420.808544:0:12020:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5b85068240. 00000100:00100000:3.0:1773941420.808545:0:12020:0:(service.c:3162:ptlrpc_main()) ldlm_cb01_000: service thread [ffff8b5b862a5400:12020] 0 exiting: rc = 1 00000020:00000010:3.0:1773941420.808580:0:12021:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b885b0c00. 00000040:00000010:3.0:1773941420.808582:0:12021:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b885b0200. 00000020:00000010:3.0:1773941420.808584:0:12021:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d90800. 00000020:00000010:3.0:1773941420.808586:0:12021:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b8849f3b8. 00000020:00000010:3.0:1773941420.808588:0:12021:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b885b1e00. 80000000:00000010:3.0:1773941420.808590:0:12021:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc92a0. 40000000:00000010:3.0:1773941420.808591:0:12021:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b85068700. 00100000:00000010:3.0:1773941420.808593:0:12021:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b90d94800. 00040000:00000010:3.0:1773941420.808595:0:12021:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b884be700. 00040000:00000010:3.0:1773941420.808596:0:12021:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b90d94000. 00040000:00000010:3.0:1773941420.808598:0:12021:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b9a587600. 00000004:00000010:3.0:1773941420.808600:0:12021:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b90d93800. 00000004:00000010:3.0:1773941420.808602:0:12021:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5ba0b89000. 00000004:00000010:3.0:1773941420.808610:0:12021:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b90d90000. 00002000:00000010:3.0:1773941420.808612:0:12021:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0b90400. 00000004:00000010:3.0:1773941420.808614:0:12021:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5ba0b90c00. 00000020:00000010:3.0:1773941420.808616:0:12021:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b885b0800. 00000020:00000010:3.0:1773941420.808619:0:12021:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068600. 00000100:00000010:3.0:1773941420.808621:0:12021:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5b85068940. 00000100:00100000:3.0:1773941420.808624:0:12021:0:(service.c:3162:ptlrpc_main()) ldlm_cb01_001: service thread [ffff8b5b862a4400:12021] 1 exiting: rc = 0 00000100:00000010:3.0:1773941420.808670:0:12100:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b862a5400. 00000100:00000010:3.0:1773941420.808676:0:12100:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b862a4400. 00000100:00000001:3.0:1773941420.808679:0:12100:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:3.0:1773941420.808680:0:12100:0:(service.c:3391:ptlrpc_stop_all_threads()) Process leaving 00000400:00000200:3.0:1773941420.808684:0:12100:0:(lib-ptl.c:969:lnet_clear_lazy_portal()) clearing portal 15 lazy 00000400:00000001:3.0:1773941420.808688:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.808690:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.808693:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034d68. 00000400:00000200:3.0:1773941420.808697:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b98115180 00000400:00000010:3.0:1773941420.808700:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b98115180. 00000100:00000001:3.0:1773941420.808728:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.808732:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_cbd 00000100:00100000:3.0:1773941420.808741:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.808743:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:3.0:1773941420.808747:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.808750:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.808751:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.808754:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87e18. 00000400:00000200:3.0:1773941420.808758:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83ea3b40 00000400:00000010:3.0:1773941420.808761:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b83ea3b40. 00000100:00000001:3.0:1773941420.808765:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.808767:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_cbd 00000100:00100000:3.0:1773941420.808774:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.808776:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:3.0:1773941420.808779:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1773941420.808783:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.808785:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:3.0:1773941420.808792:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d86000. 00000100:00000010:3.0:1773941420.808799:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8800c000. 00000100:00000010:3.0:1773941420.808805:0:12100:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b90db2000. 00000100:00000010:3.0:1773941420.808815:0:12100:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b885e6000. 00000100:00000001:3.0:1773941420.808821:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.808822:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:3.0:1773941420.808826:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d80000. 00000100:00000010:3.0:1773941420.808832:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8800d800. 00000100:00000010:3.0:1773941420.808836:0:12100:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b90d95000. 00000100:00000010:3.0:1773941420.808840:0:12100:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b90d91800. 00000100:00000001:3.0:1773941420.808845:0:12100:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:3.0:1773941420.808847:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.808850:0:12100:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941420.808853:0:12100:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941420.808855:0:12100:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff8b5b82c59940. 00000100:00000001:3.0:1773941420.808860:0:12100:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941420.808862:0:12100:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.808865:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.808867:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b85339f00. 00000100:00000001:3.0:1773941420.808870:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.808872:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.808873:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b85339840. 00000100:00000001:3.0:1773941420.808876:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.808878:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.808879:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b85339180. 00000100:00000001:3.0:1773941420.808882:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.808883:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.808885:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5ba0b97f00. 00000100:00000001:3.0:1773941420.808888:0:12100:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000100:00000001:3.0:1773941420.808890:0:12100:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:3.0:1773941420.808891:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.808893:0:12100:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941420.808895:0:12100:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941420.808896:0:12100:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff8b5b82c597c0. 00000100:00000001:3.0:1773941420.808900:0:12100:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941420.808901:0:12100:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.808903:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.808904:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5ba0b97900. 00000100:00000001:3.0:1773941420.808906:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.808907:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.808910:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5ba0b97600. 00000100:00000001:3.0:1773941420.808913:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.808921:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.808923:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5ba0b6ee40. 00000100:00000001:3.0:1773941420.808926:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.808928:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.808929:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5ba0b6e6c0. 00000100:00000001:3.0:1773941420.808932:0:12100:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000020:00000010:3.0:1773941420.809054:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 3680 at ffff8b5b90ee6000. 00000020:00000010:3.0:1773941420.809060:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5b8800e000. 00000100:00000010:3.0:1773941420.809090:0:12100:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff8b5ba0c7c000. 00000100:00000010:3.0:1773941420.809104:0:12100:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff8b5b90ee7000. 00000100:00000010:3.0:1773941420.809109:0:12100:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff8b5b856d0000. 00000100:00000010:3.0:1773941420.809116:0:12100:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff8b5b90ee3000. 00000100:00000010:3.0:1773941420.809120:0:12100:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff8b5b8800d400. 00000100:00000010:3.0:1773941420.809123:0:12100:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff8b5b8800f800. 00000100:00000010:3.0:1773941420.809126:0:12100:0:(service.c:3836:ptlrpc_service_free()) kfreed 'svc': 360 at ffff8b5b88c94c00. 00000100:00000001:3.0:1773941420.809129:0:12100:0:(service.c:3863:ptlrpc_unregister_service()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.809131:0:12100:0:(service.c:3841:ptlrpc_unregister_service()) Process entered 00000100:00000200:3.0:1773941420.809132:0:12100:0:(service.c:3843:ptlrpc_unregister_service()) ldlm_canceld: tearing down 00000100:00000001:3.0:1773941420.809136:0:12100:0:(service.c:3384:ptlrpc_stop_all_threads()) Process entered 00000100:00000001:3.0:1773941420.809138:0:12100:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 00000100:00000040:3.0:1773941420.809139:0:12100:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_canceld 00000100:00000040:3.0:1773941420.809140:0:12100:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #2 00000100:00000040:3.0:1773941420.809142:0:12100:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #1 00000100:00000040:3.0:1773941420.809144:0:12100:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #0 00000100:00000040:3.0:1773941420.809165:0:12100:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cn #2 00000020:00000010:1.0:1773941420.809171:0:12023:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b91d90000. 00000020:00000010:1.0:1773941420.809178:0:12023:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5b84cfb1e0. 00000040:00000010:1.0:1773941420.809182:0:12023:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b91d91800. 00000020:00000010:1.0:1773941420.809184:0:12023:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b99ed1800. 00000020:00000010:1.0:1773941420.809187:0:12023:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b98a04670. 00000020:00000010:1.0:1773941420.809190:0:12023:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b91d90400. 80000000:00000010:1.0:1773941420.809192:0:12023:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b84cfb9c0. 40000000:00000010:1.0:1773941420.809195:0:12023:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b82c59400. 00008000:00000010:1.0:1773941420.809198:0:12023:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d26a60. 00100000:00000010:1.0:1773941420.809201:0:12023:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b88952800. 00040000:00000010:1.0:1773941420.809203:0:12023:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5ba0c22700. 00040000:00000010:1.0:1773941420.809204:0:12023:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b88952000. 00040000:00000010:1.0:1773941420.809207:0:12023:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b8a0d1800. 00000004:00000010:1.0:1773941420.809213:0:12023:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b88957000. 00000004:00000010:1.0:1773941420.809215:0:12023:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5b85e5b000. 00000004:00000010:1.0:1773941420.809218:0:12023:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b88955800. 00002000:00000010:1.0:1773941420.809221:0:12023:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0c2ac00. 00000004:00000010:1.0:1773941420.809223:0:12023:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5ba0c29c00. 00000020:00000010:1.0:1773941420.809225:0:12023:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b839e5e00. 00000020:00000010:1.0:1773941420.809229:0:12023:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b82c59540. 00000100:00000010:1.0:1773941420.809232:0:12023:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5b82c599c0. 00000100:00100000:1.0:1773941420.809233:0:12023:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_001: service thread [ffff8b5b98183c00:12023] 1 exiting: rc = 0 00000020:00000010:0.0:1773941420.809239:0:12022:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab9800. 00000020:00000010:0.0:1773941420.809246:0:12022:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5b85ef6de0. 00000040:00000010:0.0:1773941420.809249:0:12022:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab9e00. 00000020:00000010:0.0:1773941420.809254:0:12022:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90db4800. 00000020:00000010:0.0:1773941420.809258:0:12022:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b9829e4a0. 00000020:00000010:0.0:1773941420.809262:0:12022:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab8200. 80000000:00000010:0.0:1773941420.809282:0:12022:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b85ef6300. 40000000:00000010:0.0:1773941420.809284:0:12022:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5ba039cd40. 00008000:00000010:0.0:1773941420.809287:0:12022:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90283c08. 00100000:00000010:0.0:1773941420.809291:0:12022:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b90db7800. 00040000:00000010:0.0:1773941420.809293:0:12022:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b99101c00. 00040000:00000010:0.0:1773941420.809296:0:12022:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b90db3000. 00040000:00000010:0.0:1773941420.809299:0:12022:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab9000. 00000004:00000010:0.0:1773941420.809301:0:12022:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b90db7000. 00000004:00000010:0.0:1773941420.809304:0:12022:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5b98acb000. 00000004:00000010:0.0:1773941420.809307:0:12022:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b90db5800. 00002000:00000010:0.0:1773941420.809309:0:12022:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0bf0800. 00000004:00000010:0.0:1773941420.809312:0:12022:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5ba0bf2000. 00000020:00000010:0.0:1773941420.809314:0:12022:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab8000. 00000020:00000010:0.0:1773941420.809317:0:12022:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5ba039ca00. 00000100:00000010:0.0:1773941420.809320:0:12022:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5ba039c3c0. 00000100:00100000:0.0:1773941420.809321:0:12022:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_000: service thread [ffff8b5b98183400:12022] 0 exiting: rc = 1 00000020:00000010:0.0:1773941420.809349:0:12024:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b8a0f2400. 00000020:00000010:0.0:1773941420.809352:0:12024:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5b85ef6000. 00000040:00000010:0.0:1773941420.809353:0:12024:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b893cfe00. 00000020:00000010:0.0:1773941420.809355:0:12024:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90db0800. 00000020:00000010:0.0:1773941420.809357:0:12024:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b98298f18. 00000020:00000010:0.0:1773941420.809360:0:12024:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b893cfc00. 80000000:00000010:0.0:1773941420.809361:0:12024:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b85ef6420. 40000000:00000010:0.0:1773941420.809363:0:12024:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5ba039ce80. 00008000:00000010:0.0:1773941420.809364:0:12024:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90283880. 00100000:00000010:0.0:1773941420.809366:0:12024:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b90db0000. 00040000:00000010:0.0:1773941420.809367:0:12024:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b99101b00. 00040000:00000010:0.0:1773941420.809369:0:12024:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b86885800. 00040000:00000010:0.0:1773941420.809371:0:12024:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b893cfa00. 00000004:00000010:0.0:1773941420.809373:0:12024:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b8a327000. 00000004:00000010:0.0:1773941420.809375:0:12024:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5b98ac8000. 00000004:00000010:0.0:1773941420.809377:0:12024:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b8a324800. 00002000:00000010:0.0:1773941420.809379:0:12024:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0bf1400. 00000004:00000010:0.0:1773941420.809381:0:12024:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5ba0bf1c00. 00000020:00000010:0.0:1773941420.809387:0:12024:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b91fe9200. 00000020:00000010:0.0:1773941420.809389:0:12024:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5ba039cfc0. 00000100:00000010:0.0:1773941420.809391:0:12024:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5ba039c100. 00000100:00100000:0.0:1773941420.809392:0:12024:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_002: service thread [ffff8b5b845d5c00:12024] 2 exiting: rc = 0 00000100:00000010:3.0:1773941420.809427:0:12100:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b98183400. 00000100:00000010:3.0:1773941420.809433:0:12100:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b98183c00. 00000100:00000010:3.0:1773941420.809436:0:12100:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b845d5c00. 00000100:00000001:3.0:1773941420.809439:0:12100:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:3.0:1773941420.809440:0:12100:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 00000100:00000040:3.0:1773941420.809441:0:12100:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_canceld 00000100:00000040:3.0:1773941420.809443:0:12100:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #2 00000100:00000040:3.0:1773941420.809445:0:12100:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #1 00000100:00000040:3.0:1773941420.809446:0:12100:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #0 00000100:00000040:3.0:1773941420.809468:0:12100:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cn #2 00000020:00000010:2.0:1773941420.809482:0:12025:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b84fee800. 00000020:00000010:3.0:1773941420.809486:0:12026:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b90c3ce00. 00000020:00000010:2.0:1773941420.809488:0:12025:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0166240. 00000020:00000010:3.0:1773941420.809492:0:12026:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9de0. 00000040:00000010:2.0:1773941420.809493:0:12025:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b84feee00. 00000040:00000010:3.0:1773941420.809498:0:12026:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba216be00. 00000020:00000010:2.0:1773941420.809499:0:12025:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90f27800. 00000020:00000010:2.0:1773941420.809504:0:12025:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b88558000. 00000020:00000010:3.0:1773941420.809506:0:12026:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d96000. 00000020:00000010:2.0:1773941420.809508:0:12025:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b84fee400. 00000020:00000010:3.0:1773941420.809513:0:12026:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b88498a10. 80000000:00000010:2.0:1773941420.809513:0:12025:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba01664e0. 00000020:00000010:3.0:1773941420.809520:0:12026:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba216a000. 40000000:00000010:2.0:1773941420.809520:0:12025:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b845ad1c0. 80000000:00000010:3.0:1773941420.809528:0:12026:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc97e0. 00008000:00000010:2.0:1773941420.809528:0:12025:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d774f8. 40000000:00000010:3.0:1773941420.809535:0:12026:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b85068580. 00100000:00000010:2.0:1773941420.809535:0:12025:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b90f24000. 00008000:00000010:3.0:1773941420.809542:0:12026:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b98a1c710. 00040000:00000010:2.0:1773941420.809542:0:12025:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98bb0400. 00100000:00000010:3.0:1773941420.809549:0:12026:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b90d92000. 00040000:00000010:2.0:1773941420.809549:0:12025:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b90f24800. 00040000:00000010:3.0:1773941420.809556:0:12026:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b884be600. 00040000:00000010:2.0:1773941420.809557:0:12025:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b84fee000. 00040000:00000010:3.0:1773941420.809563:0:12026:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b90d96800. 00000004:00000010:2.0:1773941420.809564:0:12025:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b90f25000. 00040000:00000010:3.0:1773941420.809569:0:12026:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba216bc00. 00000004:00000010:2.0:1773941420.809570:0:12025:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5b88905000. 00000004:00000010:3.0:1773941420.809574:0:12026:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b90d97000. 00000004:00000010:2.0:1773941420.809576:0:12025:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b90f23800. 00000004:00000010:3.0:1773941420.809579:0:12026:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5ba0b8c000. 00002000:00000010:2.0:1773941420.809581:0:12025:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5b83f0ac00. 00000004:00000010:3.0:1773941420.809585:0:12026:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b8907e800. 00000004:00000010:2.0:1773941420.809587:0:12025:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5b83f0b000. 00002000:00000010:3.0:1773941420.809592:0:12026:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0b91800. 00000020:00000010:2.0:1773941420.809593:0:12025:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84fef600. 00000004:00000010:3.0:1773941420.809598:0:12026:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5ba0b92800. 00000020:00000010:2.0:1773941420.809601:0:12025:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b845add00. 00000020:00000010:3.0:1773941420.809603:0:12026:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84ed4e00. 00000020:00000010:3.0:1773941420.809609:0:12026:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068bc0. 00000100:00000010:2.0:1773941420.809611:0:12025:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5b845add40. 00000100:00000010:3.0:1773941420.809615:0:12026:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5b850686c0. 00000100:00100000:2.0:1773941420.809617:0:12025:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_000: service thread [ffff8b5b83991e00:12025] 0 exiting: rc = 1 00000100:00100000:3.0:1773941420.809619:0:12026:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_001: service thread [ffff8b5b84ed4a00:12026] 1 exiting: rc = 0 00000020:00000010:3.0:1773941420.809679:0:12027:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b99104800. 00000020:00000010:3.0:1773941420.809683:0:12027:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9240. 00000040:00000010:3.0:1773941420.809686:0:12027:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b98162600. 00000020:00000010:3.0:1773941420.809700:0:12027:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5ba21ce800. 00000020:00000010:3.0:1773941420.809729:0:12027:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b88499420. 00000020:00000010:3.0:1773941420.809734:0:12027:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b839e2600. 80000000:00000010:3.0:1773941420.809738:0:12027:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9300. 40000000:00000010:3.0:1773941420.809741:0:12027:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b85068c40. 00008000:00000010:3.0:1773941420.809744:0:12027:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b98a1fc08. 00100000:00000010:3.0:1773941420.809747:0:12027:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b9066a800. 00040000:00000010:3.0:1773941420.809751:0:12027:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b884bec00. 00040000:00000010:3.0:1773941420.809755:0:12027:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b9066d000. 00040000:00000010:3.0:1773941420.809766:0:12027:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b839e2a00. 00000004:00000010:3.0:1773941420.809769:0:12027:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b9066e800. 00000004:00000010:3.0:1773941420.809773:0:12027:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5ba0b88000. 00000004:00000010:3.0:1773941420.809777:0:12027:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b9066a000. 00002000:00000010:3.0:1773941420.809780:0:12027:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0b93c00. 00000004:00000010:3.0:1773941420.809784:0:12027:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5ba0b91000. 00000020:00000010:3.0:1773941420.809787:0:12027:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b99105000. 00000020:00000010:3.0:1773941420.809793:0:12027:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068740. 00000100:00000010:3.0:1773941420.809797:0:12027:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5b85068d80. 00000100:00100000:3.0:1773941420.809800:0:12027:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_002: service thread [ffff8b5ba0f91200:12027] 2 exiting: rc = 0 00000100:00000010:3.0:1773941420.809836:0:12100:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b83991e00. 00000100:00000010:3.0:1773941420.809840:0:12100:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b84ed4a00. 00000100:00000010:3.0:1773941420.809843:0:12100:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5ba0f91200. 00000100:00000001:3.0:1773941420.809845:0:12100:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:3.0:1773941420.809847:0:12100:0:(service.c:3391:ptlrpc_stop_all_threads()) Process leaving 00000400:00000200:3.0:1773941420.809849:0:12100:0:(lib-ptl.c:969:lnet_clear_lazy_portal()) clearing portal 17 lazy 00000400:00000001:3.0:1773941420.809852:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.809853:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.809855:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93d68. 00000400:00000200:3.0:1773941420.809858:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b890646c0 00000400:00000010:3.0:1773941420.809860:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b890646c0. 00000100:00000001:3.0:1773941420.809863:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.809865:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.809869:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.809871:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1773941420.809873:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.809874:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.809875:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.809876:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93d10. 00000400:00000200:3.0:1773941420.809878:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b89064600 00000400:00000010:3.0:1773941420.809879:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b89064600. 00000100:00000001:3.0:1773941420.809881:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.809882:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.809886:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.809887:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 62 buffers still posted 00000100:00000001:3.0:1773941420.809889:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.809895:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.809896:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.809898:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c939f8. 00000400:00000200:3.0:1773941420.809900:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b89064e40 00000400:00000010:3.0:1773941420.809902:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b89064e40. 00000100:00000001:3.0:1773941420.809904:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.809905:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.809909:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.809910:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 61 buffers still posted 00000100:00000001:3.0:1773941420.809911:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.809913:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.809914:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.809916:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93b00. 00000400:00000200:3.0:1773941420.809918:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b89064240 00000400:00000010:3.0:1773941420.809919:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b89064240. 00000100:00000001:3.0:1773941420.809921:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.809923:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.809927:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.809928:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 60 buffers still posted 00000100:00000001:3.0:1773941420.809930:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.809931:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.809932:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.809934:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93630. 00000400:00000200:3.0:1773941420.809936:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b89064900 00000400:00000010:3.0:1773941420.809937:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b89064900. 00000100:00000001:3.0:1773941420.809939:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.809941:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.809945:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.809946:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 59 buffers still posted 00000100:00000001:3.0:1773941420.809948:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.809949:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.809951:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.809952:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93108. 00000400:00000200:3.0:1773941420.809954:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b89064a80 00000400:00000010:3.0:1773941420.809955:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b89064a80. 00000100:00000001:3.0:1773941420.809957:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.809958:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.809962:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.809970:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 58 buffers still posted 00000100:00000001:3.0:1773941420.809971:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.809974:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.809975:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.809976:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b76e0. 00000400:00000200:3.0:1773941420.809979:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b89064c00 00000400:00000010:3.0:1773941420.809980:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b89064c00. 00000100:00000001:3.0:1773941420.809982:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.809983:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.809987:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.809989:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 57 buffers still posted 00000100:00000001:3.0:1773941420.809990:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.809992:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.809993:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.809994:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b74d0. 00000400:00000200:3.0:1773941420.809996:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b89064780 00000400:00000010:3.0:1773941420.809998:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b89064780. 00000100:00000001:3.0:1773941420.809999:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810000:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810005:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810006:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 56 buffers still posted 00000100:00000001:3.0:1773941420.810007:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810009:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810010:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810011:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7688. 00000400:00000200:3.0:1773941420.810013:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84476d80 00000400:00000010:3.0:1773941420.810015:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84476d80. 00000100:00000001:3.0:1773941420.810017:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810018:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810022:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810023:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 55 buffers still posted 00000100:00000001:3.0:1773941420.810025:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810026:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810027:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810028:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b70b0. 00000400:00000200:3.0:1773941420.810031:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e300 00000400:00000010:3.0:1773941420.810032:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e300. 00000100:00000001:3.0:1773941420.810034:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810039:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810043:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810044:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 54 buffers still posted 00000100:00000001:3.0:1773941420.810046:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810048:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810049:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810051:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b72c0. 00000400:00000200:3.0:1773941420.810053:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e0c0 00000400:00000010:3.0:1773941420.810054:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941420.810057:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810058:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810062:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810063:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 53 buffers still posted 00000100:00000001:3.0:1773941420.810064:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810066:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810067:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810068:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7420. 00000400:00000200:3.0:1773941420.810070:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e9c0 00000400:00000010:3.0:1773941420.810072:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e9c0. 00000100:00000001:3.0:1773941420.810074:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810075:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810079:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810080:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 52 buffers still posted 00000100:00000001:3.0:1773941420.810083:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810084:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810085:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810087:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7f78. 00000400:00000200:3.0:1773941420.810089:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e600 00000400:00000010:3.0:1773941420.810090:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e600. 00000100:00000001:3.0:1773941420.810092:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810093:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810097:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810099:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 51 buffers still posted 00000100:00000001:3.0:1773941420.810100:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810101:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810102:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810104:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7160. 00000400:00000200:3.0:1773941420.810106:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e480 00000400:00000010:3.0:1773941420.810112:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e480. 00000100:00000001:3.0:1773941420.810115:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810116:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810119:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810121:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 50 buffers still posted 00000100:00000001:3.0:1773941420.810123:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810124:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810125:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810127:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7528. 00000400:00000200:3.0:1773941420.810129:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5ed80 00000400:00000010:3.0:1773941420.810130:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5ed80. 00000100:00000001:3.0:1773941420.810132:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810133:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810137:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810138:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 49 buffers still posted 00000100:00000001:3.0:1773941420.810139:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810141:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810142:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810143:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7318. 00000400:00000200:3.0:1773941420.810145:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e900 00000400:00000010:3.0:1773941420.810146:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e900. 00000100:00000001:3.0:1773941420.810148:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810149:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810153:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810155:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 48 buffers still posted 00000100:00000001:3.0:1773941420.810156:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810158:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810159:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810160:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b71b8. 00000400:00000200:3.0:1773941420.810162:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5eb40 00000400:00000010:3.0:1773941420.810164:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5eb40. 00000100:00000001:3.0:1773941420.810166:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810167:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810170:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810172:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 47 buffers still posted 00000100:00000001:3.0:1773941420.810174:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810175:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810182:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810183:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7478. 00000400:00000200:3.0:1773941420.810185:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5ecc0 00000400:00000010:3.0:1773941420.810187:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5ecc0. 00000100:00000001:3.0:1773941420.810189:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810191:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810195:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810197:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 46 buffers still posted 00000100:00000001:3.0:1773941420.810199:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810200:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810201:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810203:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7a50. 00000400:00000200:3.0:1773941420.810205:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5ea80 00000400:00000010:3.0:1773941420.810207:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5ea80. 00000100:00000001:3.0:1773941420.810209:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810210:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810214:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810215:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 45 buffers still posted 00000100:00000001:3.0:1773941420.810217:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810219:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810220:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810221:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7c08. 00000400:00000200:3.0:1773941420.810223:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e840 00000400:00000010:3.0:1773941420.810225:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e840. 00000100:00000001:3.0:1773941420.810227:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810228:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810232:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810233:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 44 buffers still posted 00000100:00000001:3.0:1773941420.810235:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810236:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810237:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810239:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7948. 00000400:00000200:3.0:1773941420.810241:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e540 00000400:00000010:3.0:1773941420.810242:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e540. 00000100:00000001:3.0:1773941420.810244:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810245:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810249:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810251:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 43 buffers still posted 00000100:00000001:3.0:1773941420.810258:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810260:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810261:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810263:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7d68. 00000400:00000200:3.0:1773941420.810265:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e180 00000400:00000010:3.0:1773941420.810292:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e180. 00000100:00000001:3.0:1773941420.810295:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810296:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810300:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810302:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 42 buffers still posted 00000100:00000001:3.0:1773941420.810303:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810305:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810306:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810308:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7d10. 00000400:00000200:3.0:1773941420.810310:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e000 00000400:00000010:3.0:1773941420.810311:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e000. 00000100:00000001:3.0:1773941420.810313:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810315:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810319:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810320:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 41 buffers still posted 00000100:00000001:3.0:1773941420.810322:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810323:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810324:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810326:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b79f8. 00000400:00000200:3.0:1773941420.810328:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e3c0 00000400:00000010:3.0:1773941420.810329:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e3c0. 00000100:00000001:3.0:1773941420.810331:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810332:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810336:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810337:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 40 buffers still posted 00000100:00000001:3.0:1773941420.810339:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810340:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810341:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810343:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7b00. 00000400:00000200:3.0:1773941420.810344:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5ee40 00000400:00000010:3.0:1773941420.810346:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5ee40. 00000100:00000001:3.0:1773941420.810348:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810349:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810358:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810360:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 39 buffers still posted 00000100:00000001:3.0:1773941420.810361:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810364:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810365:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810367:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7630. 00000400:00000200:3.0:1773941420.810369:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e780 00000400:00000010:3.0:1773941420.810371:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e780. 00000100:00000001:3.0:1773941420.810373:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810374:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810378:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810379:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 38 buffers still posted 00000100:00000001:3.0:1773941420.810381:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810382:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810383:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810384:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7108. 00000400:00000200:3.0:1773941420.810386:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5ef00 00000400:00000010:3.0:1773941420.810388:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5ef00. 00000100:00000001:3.0:1773941420.810390:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810391:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810395:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810396:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 37 buffers still posted 00000100:00000001:3.0:1773941420.810398:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810400:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810401:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810402:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7790. 00000400:00000200:3.0:1773941420.810404:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5ec00 00000400:00000010:3.0:1773941420.810405:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5ec00. 00000100:00000001:3.0:1773941420.810407:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810408:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810412:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810414:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 36 buffers still posted 00000100:00000001:3.0:1773941420.810415:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810417:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810418:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810419:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7738. 00000400:00000200:3.0:1773941420.810421:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e6c0 00000400:00000010:3.0:1773941420.810423:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e6c0. 00000100:00000001:3.0:1773941420.810433:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810434:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810439:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810440:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 35 buffers still posted 00000100:00000001:3.0:1773941420.810441:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810443:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810445:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810446:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7cb8. 00000400:00000200:3.0:1773941420.810449:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b98124f00 00000400:00000010:3.0:1773941420.810450:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b98124f00. 00000100:00000001:3.0:1773941420.810453:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810454:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810458:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810460:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 34 buffers still posted 00000100:00000001:3.0:1773941420.810461:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810463:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810464:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810465:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7bb0. 00000400:00000200:3.0:1773941420.810467:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b98124a80 00000400:00000010:3.0:1773941420.810469:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b98124a80. 00000100:00000001:3.0:1773941420.810470:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810472:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810476:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810477:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 33 buffers still posted 00000100:00000001:3.0:1773941420.810479:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810481:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810482:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810483:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7268. 00000400:00000200:3.0:1773941420.810485:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b98124780 00000400:00000010:3.0:1773941420.810487:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b98124780. 00000100:00000001:3.0:1773941420.810488:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810490:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810494:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810495:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 32 buffers still posted 00000100:00000001:3.0:1773941420.810497:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810498:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810499:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810501:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7370. 00000400:00000200:3.0:1773941420.810511:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b98124cc0 00000400:00000010:3.0:1773941420.810512:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b98124cc0. 00000100:00000001:3.0:1773941420.810515:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810516:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810521:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810522:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 31 buffers still posted 00000100:00000001:3.0:1773941420.810524:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810525:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810526:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810528:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7e70. 00000400:00000200:3.0:1773941420.810530:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b98124300 00000400:00000010:3.0:1773941420.810532:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b98124300. 00000100:00000001:3.0:1773941420.810534:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810535:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810539:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810540:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 30 buffers still posted 00000100:00000001:3.0:1773941420.810542:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810543:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810544:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810545:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7898. 00000400:00000200:3.0:1773941420.810547:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ede180 00000400:00000010:3.0:1773941420.810549:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ede180. 00000100:00000001:3.0:1773941420.810551:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810552:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810556:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810557:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 29 buffers still posted 00000100:00000001:3.0:1773941420.810559:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810560:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810562:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810563:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7580. 00000400:00000200:3.0:1773941420.810565:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ede300 00000400:00000010:3.0:1773941420.810567:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ede300. 00000100:00000001:3.0:1773941420.810569:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810570:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810574:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810575:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 28 buffers still posted 00000100:00000001:3.0:1773941420.810577:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810584:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810586:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810588:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7f20. 00000400:00000200:3.0:1773941420.810590:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ede6c0 00000400:00000010:3.0:1773941420.810591:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ede6c0. 00000100:00000001:3.0:1773941420.810593:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810594:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810598:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810600:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 27 buffers still posted 00000100:00000001:3.0:1773941420.810601:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810603:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810604:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810606:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7b58. 00000400:00000200:3.0:1773941420.810608:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ede840 00000400:00000010:3.0:1773941420.810609:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ede840. 00000100:00000001:3.0:1773941420.810611:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810612:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810617:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810618:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 26 buffers still posted 00000100:00000001:3.0:1773941420.810620:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810621:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810622:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810624:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b75d8. 00000400:00000200:3.0:1773941420.810626:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ede600 00000400:00000010:3.0:1773941420.810628:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ede600. 00000100:00000001:3.0:1773941420.810630:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810631:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810635:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810637:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 25 buffers still posted 00000100:00000001:3.0:1773941420.810639:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810641:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810642:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810644:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7000. 00000400:00000200:3.0:1773941420.810646:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ede240 00000400:00000010:3.0:1773941420.810648:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ede240. 00000100:00000001:3.0:1773941420.810649:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810651:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810655:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810662:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 24 buffers still posted 00000100:00000001:3.0:1773941420.810664:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810666:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810667:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810669:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b78f0. 00000400:00000200:3.0:1773941420.810671:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85edef00 00000400:00000010:3.0:1773941420.810673:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85edef00. 00000100:00000001:3.0:1773941420.810675:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810677:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810680:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810682:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 23 buffers still posted 00000100:00000001:3.0:1773941420.810683:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810685:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810686:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810687:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7840. 00000400:00000200:3.0:1773941420.810690:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ede0c0 00000400:00000010:3.0:1773941420.810691:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ede0c0. 00000100:00000001:3.0:1773941420.810693:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810694:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810698:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810699:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 22 buffers still posted 00000100:00000001:3.0:1773941420.810701:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810910:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810913:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810915:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7dc0. 00000400:00000200:3.0:1773941420.810918:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ede780 00000400:00000010:3.0:1773941420.810920:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ede780. 00000100:00000001:3.0:1773941420.810922:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810923:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810927:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810929:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 21 buffers still posted 00000100:00000001:3.0:1773941420.810931:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810932:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810934:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810935:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7ec8. 00000400:00000200:3.0:1773941420.810937:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85edea80 00000400:00000010:3.0:1773941420.810939:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85edea80. 00000100:00000001:3.0:1773941420.810941:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810947:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810951:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810952:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 20 buffers still posted 00000100:00000001:3.0:1773941420.810953:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810956:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810957:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810958:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b77e8. 00000400:00000200:3.0:1773941420.810960:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85eded80 00000400:00000010:3.0:1773941420.810962:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85eded80. 00000100:00000001:3.0:1773941420.810964:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810965:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810969:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810971:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 19 buffers still posted 00000100:00000001:3.0:1773941420.810972:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810974:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810975:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810976:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b79a0. 00000400:00000200:3.0:1773941420.810978:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85edeb40 00000400:00000010:3.0:1773941420.810979:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85edeb40. 00000100:00000001:3.0:1773941420.810981:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810982:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.810986:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.810987:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 18 buffers still posted 00000100:00000001:3.0:1773941420.810988:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.810990:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.810990:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.810992:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b73c8. 00000400:00000200:3.0:1773941420.810994:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ede3c0 00000400:00000010:3.0:1773941420.810996:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ede3c0. 00000100:00000001:3.0:1773941420.810998:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.810999:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811003:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811005:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 17 buffers still posted 00000100:00000001:3.0:1773941420.811006:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811008:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811009:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811010:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7210. 00000400:00000200:3.0:1773941420.811012:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f8b40 00000400:00000010:3.0:1773941420.811018:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f8b40. 00000100:00000001:3.0:1773941420.811021:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811022:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811026:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811028:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 16 buffers still posted 00000100:00000001:3.0:1773941420.811029:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811031:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811032:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811034:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7c60. 00000400:00000200:3.0:1773941420.811036:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f8780 00000400:00000010:3.0:1773941420.811037:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f8780. 00000100:00000001:3.0:1773941420.811039:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811040:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811044:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811046:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 15 buffers still posted 00000100:00000001:3.0:1773941420.811047:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811048:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811049:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811051:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7aa8. 00000400:00000200:3.0:1773941420.811053:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f89c0 00000400:00000010:3.0:1773941420.811054:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f89c0. 00000100:00000001:3.0:1773941420.811056:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811057:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811061:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811063:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 14 buffers still posted 00000100:00000001:3.0:1773941420.811064:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811066:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811067:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811069:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7e18. 00000400:00000200:3.0:1773941420.811071:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f8600 00000400:00000010:3.0:1773941420.811072:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f8600. 00000100:00000001:3.0:1773941420.811074:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811075:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811080:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811082:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 13 buffers still posted 00000100:00000001:3.0:1773941420.811083:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811085:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811092:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811094:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b885b7058. 00000400:00000200:3.0:1773941420.811096:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f8c00 00000400:00000010:3.0:1773941420.811098:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f8c00. 00000100:00000001:3.0:1773941420.811101:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811103:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811108:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811109:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 12 buffers still posted 00000100:00000001:3.0:1773941420.811111:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811113:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811114:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811115:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014bec8. 00000400:00000200:3.0:1773941420.811118:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f83c0 00000400:00000010:3.0:1773941420.811119:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f83c0. 00000100:00000001:3.0:1773941420.811121:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811123:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811127:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811128:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 11 buffers still posted 00000100:00000001:3.0:1773941420.811130:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811132:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811133:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811134:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014bdc0. 00000400:00000200:3.0:1773941420.811136:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f8900 00000400:00000010:3.0:1773941420.811138:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f8900. 00000100:00000001:3.0:1773941420.811139:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811141:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811144:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811146:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 10 buffers still posted 00000100:00000001:3.0:1773941420.811148:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811149:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811151:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811152:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014b840. 00000400:00000200:3.0:1773941420.811155:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f8240 00000400:00000010:3.0:1773941420.811156:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f8240. 00000100:00000001:3.0:1773941420.811158:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811159:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811163:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811165:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 9 buffers still posted 00000100:00000001:3.0:1773941420.811173:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811175:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811177:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811179:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014b8f0. 00000400:00000200:3.0:1773941420.811181:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f8840 00000400:00000010:3.0:1773941420.811183:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f8840. 00000100:00000001:3.0:1773941420.811185:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811186:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811189:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811191:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 8 buffers still posted 00000100:00000001:3.0:1773941420.811192:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811193:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811195:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811196:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014b5d8. 00000400:00000200:3.0:1773941420.811198:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f8540 00000400:00000010:3.0:1773941420.811200:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f8540. 00000100:00000001:3.0:1773941420.811202:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811203:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811207:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811209:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 7 buffers still posted 00000100:00000001:3.0:1773941420.811210:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811212:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811213:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811215:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014b000. 00000400:00000200:3.0:1773941420.811217:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f8cc0 00000400:00000010:3.0:1773941420.811218:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f8cc0. 00000100:00000001:3.0:1773941420.811220:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811221:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811225:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811226:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 6 buffers still posted 00000100:00000001:3.0:1773941420.811228:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811229:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811231:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811232:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014b9a0. 00000400:00000200:3.0:1773941420.811234:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f86c0 00000400:00000010:3.0:1773941420.811236:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f86c0. 00000100:00000001:3.0:1773941420.811238:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811239:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811248:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811250:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 5 buffers still posted 00000100:00000001:3.0:1773941420.811251:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811253:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811254:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811256:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014be70. 00000400:00000200:3.0:1773941420.811258:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f8000 00000400:00000010:3.0:1773941420.811259:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f8000. 00000100:00000001:3.0:1773941420.811261:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811262:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811279:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811281:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 4 buffers still posted 00000100:00000001:3.0:1773941420.811283:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811284:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811285:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811286:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014b898. 00000400:00000200:3.0:1773941420.811289:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f8f00 00000400:00000010:3.0:1773941420.811290:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f8f00. 00000100:00000001:3.0:1773941420.811292:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811293:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811297:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811298:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 3 buffers still posted 00000100:00000001:3.0:1773941420.811300:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811301:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811302:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811304:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014b580. 00000400:00000200:3.0:1773941420.811306:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f8480 00000400:00000010:3.0:1773941420.811308:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f8480. 00000100:00000001:3.0:1773941420.811309:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811311:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811315:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811316:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 2 buffers still posted 00000100:00000001:3.0:1773941420.811318:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811319:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811320:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811321:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014bf20. 00000400:00000200:3.0:1773941420.811323:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f8d80 00000400:00000010:3.0:1773941420.811324:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f8d80. 00000100:00000001:3.0:1773941420.811333:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811334:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811338:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811339:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 1 buffers still posted 00000100:00000001:3.0:1773941420.811341:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811343:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811344:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811346:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014bb58. 00000400:00000200:3.0:1773941420.811348:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b889f8a80 00000400:00000010:3.0:1773941420.811349:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b889f8a80. 00000100:00000001:3.0:1773941420.811351:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811352:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811356:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811357:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:3.0:1773941420.811359:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811361:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811363:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811364:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9210. 00000400:00000200:3.0:1773941420.811367:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0cfb480 00000400:00000010:3.0:1773941420.811368:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0cfb480. 00000100:00000001:3.0:1773941420.811370:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811371:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811375:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811376:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1773941420.811378:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811379:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811380:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811382:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9c60. 00000400:00000200:3.0:1773941420.811384:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0cfb540 00000400:00000010:3.0:1773941420.811385:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0cfb540. 00000100:00000001:3.0:1773941420.811387:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811388:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811393:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811394:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 62 buffers still posted 00000100:00000001:3.0:1773941420.811396:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811397:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811398:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811400:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9aa8. 00000400:00000200:3.0:1773941420.811406:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0cfb840 00000400:00000010:3.0:1773941420.811408:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0cfb840. 00000100:00000001:3.0:1773941420.811410:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811411:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811415:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811417:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 61 buffers still posted 00000100:00000001:3.0:1773941420.811418:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811420:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811421:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811422:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9e18. 00000400:00000200:3.0:1773941420.811424:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0cfb600 00000400:00000010:3.0:1773941420.811426:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0cfb600. 00000100:00000001:3.0:1773941420.811428:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811429:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811433:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811434:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 60 buffers still posted 00000100:00000001:3.0:1773941420.811435:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811437:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811438:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811439:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9058. 00000400:00000200:3.0:1773941420.811441:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0cfb240 00000400:00000010:3.0:1773941420.811443:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0cfb240. 00000100:00000001:3.0:1773941420.811445:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811446:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811450:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811451:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 59 buffers still posted 00000100:00000001:3.0:1773941420.811453:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811454:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811455:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811457:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af96e0. 00000400:00000200:3.0:1773941420.811459:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0cfb300 00000400:00000010:3.0:1773941420.811461:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0cfb300. 00000100:00000001:3.0:1773941420.811463:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811464:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811468:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811469:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 58 buffers still posted 00000100:00000001:3.0:1773941420.811470:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811477:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811478:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811480:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af94d0. 00000400:00000200:3.0:1773941420.811482:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0cfb000 00000400:00000010:3.0:1773941420.811483:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0cfb000. 00000100:00000001:3.0:1773941420.811485:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811486:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811490:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811491:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 57 buffers still posted 00000100:00000001:3.0:1773941420.811492:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811494:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811494:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811496:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9688. 00000400:00000200:3.0:1773941420.811498:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0cfbe40 00000400:00000010:3.0:1773941420.811499:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0cfbe40. 00000100:00000001:3.0:1773941420.811501:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811502:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811506:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811507:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 56 buffers still posted 00000100:00000001:3.0:1773941420.811509:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811510:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811511:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811513:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af90b0. 00000400:00000200:3.0:1773941420.811514:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0cfbb40 00000400:00000010:3.0:1773941420.811516:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0cfbb40. 00000100:00000001:3.0:1773941420.811518:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811519:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811522:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811524:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 55 buffers still posted 00000100:00000001:3.0:1773941420.811525:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811526:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811527:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811529:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af92c0. 00000400:00000200:3.0:1773941420.811531:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0cfbd80 00000400:00000010:3.0:1773941420.811532:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0cfbd80. 00000100:00000001:3.0:1773941420.811534:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811535:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811539:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811546:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 54 buffers still posted 00000100:00000001:3.0:1773941420.811547:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811550:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811551:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811553:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9420. 00000400:00000200:3.0:1773941420.811555:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0cfbcc0 00000400:00000010:3.0:1773941420.811557:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0cfbcc0. 00000100:00000001:3.0:1773941420.811559:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811560:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811564:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811566:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 53 buffers still posted 00000100:00000001:3.0:1773941420.811567:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811569:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811570:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811571:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9f78. 00000400:00000200:3.0:1773941420.811573:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0cfba80 00000400:00000010:3.0:1773941420.811575:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0cfba80. 00000100:00000001:3.0:1773941420.811577:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811578:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811582:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811584:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 52 buffers still posted 00000100:00000001:3.0:1773941420.811585:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811587:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811588:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811590:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9160. 00000400:00000200:3.0:1773941420.811592:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b300 00000400:00000010:3.0:1773941420.811593:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b300. 00000100:00000001:3.0:1773941420.811596:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811598:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811602:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811603:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 51 buffers still posted 00000100:00000001:3.0:1773941420.811605:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811606:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811607:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811608:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9528. 00000400:00000200:3.0:1773941420.811610:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680ba80 00000400:00000010:3.0:1773941420.811612:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680ba80. 00000100:00000001:3.0:1773941420.811614:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811620:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811624:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811625:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 50 buffers still posted 00000100:00000001:3.0:1773941420.811626:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811629:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811630:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811631:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9318. 00000400:00000200:3.0:1773941420.811634:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b240 00000400:00000010:3.0:1773941420.811635:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b240. 00000100:00000001:3.0:1773941420.811638:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811639:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811642:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811644:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 49 buffers still posted 00000100:00000001:3.0:1773941420.811645:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811647:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811648:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811649:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af91b8. 00000400:00000200:3.0:1773941420.811652:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b900 00000400:00000010:3.0:1773941420.811653:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b900. 00000100:00000001:3.0:1773941420.811655:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811656:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811661:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811662:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 48 buffers still posted 00000100:00000001:3.0:1773941420.811664:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811666:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811667:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811669:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9478. 00000400:00000200:3.0:1773941420.811671:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b9c0 00000400:00000010:3.0:1773941420.811672:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b9c0. 00000100:00000001:3.0:1773941420.811674:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811676:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811680:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811681:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 47 buffers still posted 00000100:00000001:3.0:1773941420.811682:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811684:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811685:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811686:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9a50. 00000400:00000200:3.0:1773941420.811688:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680bc00 00000400:00000010:3.0:1773941420.811696:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680bc00. 00000100:00000001:3.0:1773941420.811698:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811700:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811722:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811725:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 46 buffers still posted 00000100:00000001:3.0:1773941420.811727:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811730:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811732:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811733:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab3c8. 00000400:00000200:3.0:1773941420.811737:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680bf00 00000400:00000010:3.0:1773941420.811739:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680bf00. 00000100:00000001:3.0:1773941420.811741:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811743:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811749:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811751:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 45 buffers still posted 00000100:00000001:3.0:1773941420.811753:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811755:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811756:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811757:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab9a0. 00000400:00000200:3.0:1773941420.811759:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680bcc0 00000400:00000010:3.0:1773941420.811760:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680bcc0. 00000100:00000001:3.0:1773941420.811763:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811764:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811768:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811769:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 44 buffers still posted 00000100:00000001:3.0:1773941420.811771:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811772:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811773:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811775:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab7e8. 00000400:00000200:3.0:1773941420.811777:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b3c0 00000400:00000010:3.0:1773941420.811778:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b3c0. 00000100:00000001:3.0:1773941420.811780:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811781:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811785:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811787:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 43 buffers still posted 00000100:00000001:3.0:1773941420.811788:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811789:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811795:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811796:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abec8. 00000400:00000200:3.0:1773941420.811798:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680be40 00000400:00000010:3.0:1773941420.811800:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680be40. 00000100:00000001:3.0:1773941420.811803:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811804:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811809:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811811:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 42 buffers still posted 00000100:00000001:3.0:1773941420.811813:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811815:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811816:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811817:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abdc0. 00000400:00000200:3.0:1773941420.811819:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b0c0 00000400:00000010:3.0:1773941420.811821:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b0c0. 00000100:00000001:3.0:1773941420.811823:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811824:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811828:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811829:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 41 buffers still posted 00000100:00000001:3.0:1773941420.811831:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811832:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811833:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811835:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab210. 00000400:00000200:3.0:1773941420.811836:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b180 00000400:00000010:3.0:1773941420.811838:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b180. 00000100:00000001:3.0:1773941420.811840:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811841:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811845:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811847:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 40 buffers still posted 00000100:00000001:3.0:1773941420.811848:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811850:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811851:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811853:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abc60. 00000400:00000200:3.0:1773941420.811854:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b780 00000400:00000010:3.0:1773941420.811856:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b780. 00000100:00000001:3.0:1773941420.811858:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811859:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811863:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811865:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 39 buffers still posted 00000100:00000001:3.0:1773941420.811870:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811872:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811873:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811875:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abaa8. 00000400:00000200:3.0:1773941420.811877:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680bb40 00000400:00000010:3.0:1773941420.811878:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680bb40. 00000100:00000001:3.0:1773941420.811880:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811881:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811885:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811886:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 38 buffers still posted 00000100:00000001:3.0:1773941420.811888:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811889:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811890:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811892:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abe18. 00000400:00000200:3.0:1773941420.811894:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b480 00000400:00000010:3.0:1773941420.811895:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b480. 00000100:00000001:3.0:1773941420.811897:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811898:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811902:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811903:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 37 buffers still posted 00000100:00000001:3.0:1773941420.811904:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811906:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811907:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811908:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab058. 00000400:00000200:3.0:1773941420.811910:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7a80 00000400:00000010:3.0:1773941420.811912:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7a80. 00000100:00000001:3.0:1773941420.811913:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811915:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811918:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811920:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 36 buffers still posted 00000100:00000001:3.0:1773941420.811921:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811923:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811924:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811925:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941420.811927:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed73c0 00000400:00000010:3.0:1773941420.811928:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed73c0. 00000100:00000001:3.0:1773941420.811930:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811932:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811940:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811942:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 35 buffers still posted 00000100:00000001:3.0:1773941420.811943:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811945:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811946:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811948:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab4d0. 00000400:00000200:3.0:1773941420.811950:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7600 00000400:00000010:3.0:1773941420.811951:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7600. 00000100:00000001:3.0:1773941420.811953:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811954:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811958:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811960:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 34 buffers still posted 00000100:00000001:3.0:1773941420.811961:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811962:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811963:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811965:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab688. 00000400:00000200:3.0:1773941420.811966:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7cc0 00000400:00000010:3.0:1773941420.811968:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7cc0. 00000100:00000001:3.0:1773941420.811970:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811971:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811975:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811976:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 33 buffers still posted 00000100:00000001:3.0:1773941420.811977:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811979:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811980:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811981:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab0b0. 00000400:00000200:3.0:1773941420.811982:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed76c0 00000400:00000010:3.0:1773941420.811983:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed76c0. 00000100:00000001:3.0:1773941420.811985:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.811986:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.811990:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.811991:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 32 buffers still posted 00000100:00000001:3.0:1773941420.811993:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.811994:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.811995:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.811997:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab2c0. 00000400:00000200:3.0:1773941420.811999:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7900 00000400:00000010:3.0:1773941420.812001:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7900. 00000100:00000001:3.0:1773941420.812007:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812008:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812011:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812013:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 31 buffers still posted 00000100:00000001:3.0:1773941420.812014:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812016:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812017:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812018:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab420. 00000400:00000200:3.0:1773941420.812020:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7f00 00000400:00000010:3.0:1773941420.812021:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7f00. 00000100:00000001:3.0:1773941420.812023:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812024:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812027:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812029:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 30 buffers still posted 00000100:00000001:3.0:1773941420.812031:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812032:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812033:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812034:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abf78. 00000400:00000200:3.0:1773941420.812036:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7e40 00000400:00000010:3.0:1773941420.812038:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7e40. 00000100:00000001:3.0:1773941420.812039:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812040:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812044:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812045:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 29 buffers still posted 00000100:00000001:3.0:1773941420.812047:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812048:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812049:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812050:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab160. 00000400:00000200:3.0:1773941420.812052:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed79c0 00000400:00000010:3.0:1773941420.812054:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed79c0. 00000100:00000001:3.0:1773941420.812056:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812057:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812060:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812062:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 28 buffers still posted 00000100:00000001:3.0:1773941420.812063:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812065:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812066:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812067:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab528. 00000400:00000200:3.0:1773941420.812074:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7000 00000400:00000010:3.0:1773941420.812076:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7000. 00000100:00000001:3.0:1773941420.812078:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812079:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812082:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812084:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 27 buffers still posted 00000100:00000001:3.0:1773941420.812085:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812087:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812088:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812089:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab318. 00000400:00000200:3.0:1773941420.812090:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7840 00000400:00000010:3.0:1773941420.812091:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7840. 00000100:00000001:3.0:1773941420.812093:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812094:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812097:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812098:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 26 buffers still posted 00000100:00000001:3.0:1773941420.812099:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812100:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812101:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812102:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab1b8. 00000400:00000200:3.0:1773941420.812104:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7b40 00000400:00000010:3.0:1773941420.812105:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7b40. 00000100:00000001:3.0:1773941420.812107:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812108:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812111:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812113:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 25 buffers still posted 00000100:00000001:3.0:1773941420.812114:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812116:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812117:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812118:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab478. 00000400:00000200:3.0:1773941420.812120:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7240 00000400:00000010:3.0:1773941420.812122:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7240. 00000100:00000001:3.0:1773941420.812123:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812124:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812128:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812129:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 24 buffers still posted 00000100:00000001:3.0:1773941420.812131:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812137:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812138:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812139:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845aba50. 00000400:00000200:3.0:1773941420.812141:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7300 00000400:00000010:3.0:1773941420.812143:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7300. 00000100:00000001:3.0:1773941420.812145:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812146:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812149:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812150:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 23 buffers still posted 00000100:00000001:3.0:1773941420.812152:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812153:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812154:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812155:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abc08. 00000400:00000200:3.0:1773941420.812157:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7c00 00000400:00000010:3.0:1773941420.812158:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7c00. 00000100:00000001:3.0:1773941420.812160:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812161:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812165:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812167:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 22 buffers still posted 00000100:00000001:3.0:1773941420.812168:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812170:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812171:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812173:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab948. 00000400:00000200:3.0:1773941420.812174:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7180 00000400:00000010:3.0:1773941420.812176:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7180. 00000100:00000001:3.0:1773941420.812178:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812179:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812182:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812184:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 21 buffers still posted 00000100:00000001:3.0:1773941420.812185:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812186:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812187:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812188:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abd68. 00000400:00000200:3.0:1773941420.812190:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7d80 00000400:00000010:3.0:1773941420.812192:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7d80. 00000100:00000001:3.0:1773941420.812193:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812195:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812198:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812204:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 20 buffers still posted 00000100:00000001:3.0:1773941420.812205:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812207:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812208:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812209:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abd10. 00000400:00000200:3.0:1773941420.812211:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7780 00000400:00000010:3.0:1773941420.812213:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7780. 00000100:00000001:3.0:1773941420.812214:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812215:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812219:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812220:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 19 buffers still posted 00000100:00000001:3.0:1773941420.812222:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812223:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812224:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812225:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab9f8. 00000400:00000200:3.0:1773941420.812227:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7540 00000400:00000010:3.0:1773941420.812229:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7540. 00000100:00000001:3.0:1773941420.812230:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812231:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812235:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812236:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 18 buffers still posted 00000100:00000001:3.0:1773941420.812237:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812239:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812239:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812241:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abb00. 00000400:00000200:3.0:1773941420.812243:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7480 00000400:00000010:3.0:1773941420.812244:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7480. 00000100:00000001:3.0:1773941420.812246:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812247:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812250:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812252:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 17 buffers still posted 00000100:00000001:3.0:1773941420.812253:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812255:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812256:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812257:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab630. 00000400:00000200:3.0:1773941420.812259:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed70c0 00000400:00000010:3.0:1773941420.812261:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed70c0. 00000100:00000001:3.0:1773941420.812263:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812280:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812284:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812285:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 16 buffers still posted 00000100:00000001:3.0:1773941420.812286:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812288:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812289:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812290:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab108. 00000400:00000200:3.0:1773941420.812291:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b88ca6cc0 00000400:00000010:3.0:1773941420.812292:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b88ca6cc0. 00000100:00000001:3.0:1773941420.812294:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812295:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812297:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812298:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 15 buffers still posted 00000100:00000001:3.0:1773941420.812299:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812300:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812301:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812301:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab790. 00000400:00000200:3.0:1773941420.812302:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d719c0 00000400:00000010:3.0:1773941420.812303:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90d719c0. 00000100:00000001:3.0:1773941420.812304:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812305:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812307:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812308:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 14 buffers still posted 00000100:00000001:3.0:1773941420.812308:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812309:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812310:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812310:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab738. 00000400:00000200:3.0:1773941420.812311:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d71c00 00000400:00000010:3.0:1773941420.812312:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90d71c00. 00000100:00000001:3.0:1773941420.812313:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812314:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812316:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812317:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 13 buffers still posted 00000100:00000001:3.0:1773941420.812318:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812318:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812319:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812319:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abcb8. 00000400:00000200:3.0:1773941420.812320:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d71180 00000400:00000010:3.0:1773941420.812324:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90d71180. 00000100:00000001:3.0:1773941420.812326:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812327:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812329:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812330:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 12 buffers still posted 00000100:00000001:3.0:1773941420.812332:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812333:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812333:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812334:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abbb0. 00000400:00000200:3.0:1773941420.812335:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d71a80 00000400:00000010:3.0:1773941420.812336:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90d71a80. 00000100:00000001:3.0:1773941420.812337:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812337:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812340:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812340:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 11 buffers still posted 00000100:00000001:3.0:1773941420.812341:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812342:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812342:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812343:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab268. 00000400:00000200:3.0:1773941420.812344:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d716c0 00000400:00000010:3.0:1773941420.812345:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90d716c0. 00000100:00000001:3.0:1773941420.812346:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812346:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812349:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812349:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 10 buffers still posted 00000100:00000001:3.0:1773941420.812350:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812351:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812352:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812352:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab370. 00000400:00000200:3.0:1773941420.812353:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d71900 00000400:00000010:3.0:1773941420.812354:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90d71900. 00000100:00000001:3.0:1773941420.812355:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812356:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812358:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812359:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 9 buffers still posted 00000100:00000001:3.0:1773941420.812360:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812360:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812364:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812365:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abe70. 00000400:00000200:3.0:1773941420.812366:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d71600 00000400:00000010:3.0:1773941420.812367:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90d71600. 00000100:00000001:3.0:1773941420.812368:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812369:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812371:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812372:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 8 buffers still posted 00000100:00000001:3.0:1773941420.812373:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812373:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812374:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812375:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab898. 00000400:00000200:3.0:1773941420.812376:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d71cc0 00000400:00000010:3.0:1773941420.812376:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90d71cc0. 00000100:00000001:3.0:1773941420.812377:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812378:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812380:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812381:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 7 buffers still posted 00000100:00000001:3.0:1773941420.812382:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812382:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812383:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812384:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab580. 00000400:00000200:3.0:1773941420.812385:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d71f00 00000400:00000010:3.0:1773941420.812386:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90d71f00. 00000100:00000001:3.0:1773941420.812387:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812387:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812389:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812390:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 6 buffers still posted 00000100:00000001:3.0:1773941420.812391:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812392:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812393:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812394:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abf20. 00000400:00000200:3.0:1773941420.812394:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d71000 00000400:00000010:3.0:1773941420.812395:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90d71000. 00000100:00000001:3.0:1773941420.812396:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812397:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812399:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812400:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 5 buffers still posted 00000100:00000001:3.0:1773941420.812406:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812408:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812410:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812411:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abb58. 00000400:00000200:3.0:1773941420.812413:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d71e40 00000400:00000010:3.0:1773941420.812415:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90d71e40. 00000100:00000001:3.0:1773941420.812416:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812418:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812421:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812423:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 4 buffers still posted 00000100:00000001:3.0:1773941420.812424:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812425:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812426:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812427:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab5d8. 00000400:00000200:3.0:1773941420.812430:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d71840 00000400:00000010:3.0:1773941420.812431:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90d71840. 00000100:00000001:3.0:1773941420.812432:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812434:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812438:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812439:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 3 buffers still posted 00000100:00000001:3.0:1773941420.812441:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812442:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812443:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812445:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab000. 00000400:00000200:3.0:1773941420.812447:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d710c0 00000400:00000010:3.0:1773941420.812448:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90d710c0. 00000100:00000001:3.0:1773941420.812451:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812452:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812456:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812458:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 2 buffers still posted 00000100:00000001:3.0:1773941420.812459:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812460:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812462:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812463:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab8f0. 00000400:00000200:3.0:1773941420.812465:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d71240 00000400:00000010:3.0:1773941420.812467:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90d71240. 00000100:00000001:3.0:1773941420.812469:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812470:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812481:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812482:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 1 buffers still posted 00000100:00000001:3.0:1773941420.812483:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941420.812486:0:12100:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941420.812487:0:12100:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941420.812489:0:12100:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab840. 00000400:00000200:3.0:1773941420.812491:0:12100:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90d71540 00000400:00000010:3.0:1773941420.812492:0:12100:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90d71540. 00000100:00000001:3.0:1773941420.812495:0:12100:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941420.812496:0:12100:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941420.812499:0:12100:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941420.812501:0:12100:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:3.0:1773941420.812503:0:12100:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1773941420.812506:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812508:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812511:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812512:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812513:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812514:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812516:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812517:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812519:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812520:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812522:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812523:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812525:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812526:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812527:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812529:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812530:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812531:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812532:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812534:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812536:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812537:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812538:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812539:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812540:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812541:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812549:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812550:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812552:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812553:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812554:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812555:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812556:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812557:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812559:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812560:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812561:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812562:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812564:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812565:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812566:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812567:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812569:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812570:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812572:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812572:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812575:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812576:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812577:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812578:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812580:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812581:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812582:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812583:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812584:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812586:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812587:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812588:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812589:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812590:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812591:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812592:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812594:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812594:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812596:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812597:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812604:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812606:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812607:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812608:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812610:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812611:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812612:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812613:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812615:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812616:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812618:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812619:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812621:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812622:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812624:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812625:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812626:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812627:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812628:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812629:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812631:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812632:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812633:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812634:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812636:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812637:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812639:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812640:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812642:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812643:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812645:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812646:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812647:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812649:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812650:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812651:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812652:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812653:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812655:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812656:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812663:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812664:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812665:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812666:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812667:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812668:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812670:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812671:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812672:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812673:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812675:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812676:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812677:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812678:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812679:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812680:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812681:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812682:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812683:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812685:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.812686:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.812687:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:3.0:1773941420.812691:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d84000. 00000100:00000010:3.0:1773941420.812695:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88cb1c00. 00000100:00000010:3.0:1773941420.812698:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d82000. 00000100:00000010:3.0:1773941420.812701:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88cb3800. 00000100:00000010:3.0:1773941420.812734:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b981a0000. 00000100:00000010:3.0:1773941420.812739:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0dc5800. 00000100:00000010:3.0:1773941420.812742:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b981a2000. 00000100:00000010:3.0:1773941420.812745:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0dc5400. 00000100:00000010:3.0:1773941420.812747:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b981a6000. 00000100:00000010:3.0:1773941420.812750:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0dc5c00. 00000100:00000010:3.0:1773941420.812752:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b83d0a000. 00000100:00000010:3.0:1773941420.812771:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0dc4800. 00000100:00000010:3.0:1773941420.812775:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b7c000. 00000100:00000010:3.0:1773941420.812777:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8902ac00. 00000100:00000010:3.0:1773941420.812780:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b7e000. 00000100:00000010:3.0:1773941420.812788:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f3b800. 00000100:00000010:3.0:1773941420.812797:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b78000. 00000100:00000010:3.0:1773941420.812800:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f39000. 00000100:00000010:3.0:1773941420.812803:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98af0000. 00000100:00000010:3.0:1773941420.812805:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f39800. 00000100:00000010:3.0:1773941420.812807:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98af2000. 00000100:00000010:3.0:1773941420.812811:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f3bc00. 00000100:00000010:3.0:1773941420.812813:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98af6000. 00000100:00000010:3.0:1773941420.812815:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f38800. 00000100:00000010:3.0:1773941420.812818:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98af4000. 00000100:00000010:3.0:1773941420.812820:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f38400. 00000100:00000010:3.0:1773941420.812822:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98aea000. 00000100:00000010:3.0:1773941420.812831:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f3ac00. 00000100:00000010:3.0:1773941420.812835:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98aee000. 00000100:00000010:3.0:1773941420.812838:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f3a800. 00000100:00000010:3.0:1773941420.812840:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98aec000. 00000100:00000010:3.0:1773941420.812842:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f3a400. 00000100:00000010:3.0:1773941420.812844:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98ae8000. 00000100:00000010:3.0:1773941420.812847:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b99ef4800. 00000100:00000010:3.0:1773941420.812849:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b83cdc000. 00000100:00000010:3.0:1773941420.812852:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b99ef7400. 00000100:00000010:3.0:1773941420.812854:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b83cde000. 00000100:00000010:3.0:1773941420.812857:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b99ef4000. 00000100:00000010:3.0:1773941420.812859:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b83cda000. 00000100:00000010:3.0:1773941420.812861:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b890c2000. 00000100:00000010:3.0:1773941420.812864:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b83cd8000. 00000100:00000010:3.0:1773941420.812867:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b890c0800. 00000100:00000010:3.0:1773941420.812869:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b829e2000. 00000100:00000010:3.0:1773941420.812872:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b890c3000. 00000100:00000010:3.0:1773941420.812874:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b829e4000. 00000100:00000010:3.0:1773941420.812877:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91e49000. 00000100:00000010:3.0:1773941420.812879:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b832ac000. 00000100:00000010:3.0:1773941420.812892:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91e4a400. 00000100:00000010:3.0:1773941420.812900:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b832a8000. 00000100:00000010:3.0:1773941420.812902:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91c4fc00. 00000100:00000010:3.0:1773941420.812905:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b832aa000. 00000100:00000010:3.0:1773941420.812907:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91c4d000. 00000100:00000010:3.0:1773941420.812910:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b91cac000. 00000100:00000010:3.0:1773941420.812912:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91c4d400. 00000100:00000010:3.0:1773941420.812914:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b91ca8000. 00000100:00000010:3.0:1773941420.812918:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88011400. 00000100:00000010:3.0:1773941420.812921:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98018000. 00000100:00000010:3.0:1773941420.812923:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88012000. 00000100:00000010:3.0:1773941420.812926:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b9801c000. 00000100:00000010:3.0:1773941420.812928:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88011c00. 00000100:00000010:3.0:1773941420.812931:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b9801e000. 00000100:00000010:3.0:1773941420.812933:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88012800. 00000100:00000010:3.0:1773941420.812935:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b9801a000. 00000100:00000010:3.0:1773941420.812937:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88011000. 00000100:00000010:3.0:1773941420.812939:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b850d0000. 00000100:00000010:3.0:1773941420.812948:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88010800. 00000100:00000010:3.0:1773941420.812951:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b850d4000. 00000100:00000010:3.0:1773941420.812954:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88012400. 00000100:00000010:3.0:1773941420.812956:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b850d6000. 00000100:00000010:3.0:1773941420.812958:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88011800. 00000100:00000010:3.0:1773941420.812960:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b850d2000. 00000100:00000010:3.0:1773941420.812962:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91f8e400. 00000100:00000010:3.0:1773941420.812964:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b9028c000. 00000100:00000010:3.0:1773941420.812967:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91f8f400. 00000100:00000010:3.0:1773941420.812970:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b9028e000. 00000100:00000010:3.0:1773941420.812972:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91f8e800. 00000100:00000010:3.0:1773941420.812974:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b9028a000. 00000100:00000010:3.0:1773941420.812977:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91f8cc00. 00000100:00000010:3.0:1773941420.812979:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90288000. 00000100:00000010:3.0:1773941420.812981:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91f8d400. 00000100:00000010:3.0:1773941420.812984:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88934000. 00000100:00000010:3.0:1773941420.812993:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91f8ec00. 00000100:00000010:3.0:1773941420.812995:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88936000. 00000100:00000010:3.0:1773941420.812998:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8801c800. 00000100:00000010:3.0:1773941420.813001:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88932000. 00000100:00000010:3.0:1773941420.813003:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85613800. 00000100:00000010:3.0:1773941420.813005:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88930000. 00000100:00000010:3.0:1773941420.813007:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85610c00. 00000100:00000010:3.0:1773941420.813009:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88cfc000. 00000100:00000010:3.0:1773941420.813025:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85611000. 00000100:00000010:3.0:1773941420.813028:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88cfe000. 00000100:00000010:3.0:1773941420.813031:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5bc2069400. 00000100:00000010:3.0:1773941420.813033:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88cfa000. 00000100:00000010:3.0:1773941420.813036:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5bc2069c00. 00000100:00000010:3.0:1773941420.813038:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88cf8000. 00000100:00000010:3.0:1773941420.813041:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5bc206b000. 00000100:00000010:3.0:1773941420.813043:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b82c74000. 00000100:00000010:3.0:1773941420.813046:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5bc2068000. 00000100:00000010:3.0:1773941420.813048:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b82c76000. 00000100:00000010:3.0:1773941420.813050:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8a3a2c00. 00000100:00000010:3.0:1773941420.813052:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b82c72000. 00000100:00000010:3.0:1773941420.813055:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5bc2076000. 00000100:00000010:3.0:1773941420.813058:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b82c70000. 00000100:00000010:3.0:1773941420.813060:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5bc2073400. 00000100:00000010:3.0:1773941420.813063:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88940000. 00000100:00000010:3.0:1773941420.813065:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5bc2072000. 00000100:00000010:3.0:1773941420.813067:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88944000. 00000100:00000010:3.0:1773941420.813070:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b99f55000. 00000100:00000010:3.0:1773941420.813073:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88946000. 00000100:00000010:3.0:1773941420.813075:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b99f54400. 00000100:00000010:3.0:1773941420.813077:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88942000. 00000100:00000010:3.0:1773941420.813080:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b99f57800. 00000100:00000010:3.0:1773941420.813082:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90da4000. 00000100:00000010:3.0:1773941420.813098:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b99f56800. 00000100:00000010:3.0:1773941420.813105:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90da6000. 00000100:00000010:3.0:1773941420.813107:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91e15000. 00000100:00000010:3.0:1773941420.813111:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90da2000. 00000100:00000010:3.0:1773941420.813113:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b885acc00. 00000100:00000010:3.0:1773941420.813115:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90da0000. 00000100:00000010:3.0:1773941420.813117:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b885aec00. 00000100:00000010:3.0:1773941420.813120:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b1e000. 00000100:00000010:3.0:1773941420.813123:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b885adc00. 00000100:00000010:3.0:1773941420.813125:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b1a000. 00000100:00000010:3.0:1773941420.813127:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b885ad400. 00000100:00000010:3.0:1773941420.813129:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b18000. 00000100:00000010:3.0:1773941420.813131:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b885afc00. 00000100:00000010:3.0:1773941420.813133:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b1c000. 00000100:00000010:3.0:1773941420.813136:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b885ac800. 00000100:00000010:3.0:1773941420.813157:0:12100:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5ba0b0d800. 00000100:00000010:3.0:1773941420.813161:0:12100:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b88953000. 00000100:00000010:3.0:1773941420.813164:0:12100:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b90db1800. 00000100:00000001:3.0:1773941420.813166:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813167:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813169:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813170:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813172:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813173:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813174:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813175:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813176:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813177:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813178:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813180:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813181:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813182:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813183:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813185:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813186:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813187:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813188:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813194:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813196:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813197:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813198:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813199:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813201:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813202:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813203:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813204:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813206:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813207:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813208:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813209:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813210:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813211:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813213:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813214:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813215:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813216:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813217:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813218:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813220:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813221:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813222:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813223:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813224:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813225:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813227:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813228:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813229:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813230:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813231:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813232:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813234:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813235:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813237:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813237:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813239:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813240:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813242:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813248:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813250:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813251:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813253:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813254:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813255:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813256:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813258:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813259:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813260:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813261:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813263:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813264:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813279:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813280:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813281:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813283:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813284:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813285:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813286:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813288:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813289:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813290:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813291:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813292:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813294:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813295:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813297:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813298:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813299:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813300:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813301:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813302:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813304:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813305:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813306:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813307:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813308:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813309:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813311:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813317:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813319:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813320:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813321:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813322:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813324:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813325:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813326:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813327:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813329:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813330:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813332:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813333:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813335:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813336:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813337:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813339:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813341:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813342:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813343:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813344:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813345:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813346:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813348:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813349:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813350:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813351:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941420.813353:0:12100:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941420.813354:0:12100:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:3.0:1773941420.813358:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a12000. 00000100:00000010:3.0:1773941420.813362:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b885af800. 00000100:00000010:3.0:1773941420.813365:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a10000. 00000100:00000010:3.0:1773941420.813367:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b885ac400. 00000100:00000010:3.0:1773941420.813369:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a14000. 00000100:00000010:3.0:1773941420.813372:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b885ad000. 00000100:00000010:3.0:1773941420.813374:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a16000. 00000100:00000010:3.0:1773941420.813377:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b885ae400. 00000100:00000010:3.0:1773941420.813379:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d6c000. 00000100:00000010:3.0:1773941420.813401:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b885ae800. 00000100:00000010:3.0:1773941420.813404:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d6e000. 00000100:00000010:3.0:1773941420.813406:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b885ae000. 00000100:00000010:3.0:1773941420.813409:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d6a000. 00000100:00000010:3.0:1773941420.813411:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b885af400. 00000100:00000010:3.0:1773941420.813413:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d68000. 00000100:00000010:3.0:1773941420.813416:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b885ac000. 00000100:00000010:3.0:1773941420.813418:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a3a000. 00000100:00000010:3.0:1773941420.813421:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b885af000. 00000100:00000010:3.0:1773941420.813423:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a38000. 00000100:00000010:3.0:1773941420.813426:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90237c00. 00000100:00000010:3.0:1773941420.813429:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a3c000. 00000100:00000010:3.0:1773941420.813432:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90234800. 00000100:00000010:3.0:1773941420.813434:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a3e000. 00000100:00000010:3.0:1773941420.813436:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90235800. 00000100:00000010:3.0:1773941420.813439:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88994000. 00000100:00000010:3.0:1773941420.813443:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90237800. 00000100:00000010:3.0:1773941420.813445:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88996000. 00000100:00000010:3.0:1773941420.813449:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90234400. 00000100:00000010:3.0:1773941420.813451:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88992000. 00000100:00000010:3.0:1773941420.813454:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90235000. 00000100:00000010:3.0:1773941420.813456:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88990000. 00000100:00000010:3.0:1773941420.813459:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90236400. 00000100:00000010:3.0:1773941420.813462:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889f2000. 00000100:00000010:3.0:1773941420.813478:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90236800. 00000100:00000010:3.0:1773941420.813480:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889f0000. 00000100:00000010:3.0:1773941420.813483:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90236000. 00000100:00000010:3.0:1773941420.813484:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889f4000. 00000100:00000010:3.0:1773941420.813487:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90237400. 00000100:00000010:3.0:1773941420.813489:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889f6000. 00000100:00000010:3.0:1773941420.813492:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90234000. 00000100:00000010:3.0:1773941420.813494:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b8505a000. 00000100:00000010:3.0:1773941420.813496:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90237000. 00000100:00000010:3.0:1773941420.813504:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b85058000. 00000100:00000010:3.0:1773941420.813506:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90234c00. 00000100:00000010:3.0:1773941420.813508:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b8505c000. 00000100:00000010:3.0:1773941420.813510:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90236c00. 00000100:00000010:3.0:1773941420.813512:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b8505e000. 00000100:00000010:3.0:1773941420.813515:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90235c00. 00000100:00000010:3.0:1773941420.813516:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b9022a000. 00000100:00000010:3.0:1773941420.813521:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90235400. 00000100:00000010:3.0:1773941420.813523:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90228000. 00000100:00000010:3.0:1773941420.813525:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85071c00. 00000100:00000010:3.0:1773941420.813528:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b9022c000. 00000100:00000010:3.0:1773941420.813531:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85071400. 00000100:00000010:3.0:1773941420.813533:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b9022e000. 00000100:00000010:3.0:1773941420.813536:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85073c00. 00000100:00000010:3.0:1773941420.813538:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885f0000. 00000100:00000010:3.0:1773941420.813557:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85070800. 00000100:00000010:3.0:1773941420.813560:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885f4000. 00000100:00000010:3.0:1773941420.813563:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85071800. 00000100:00000010:3.0:1773941420.813565:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885f6000. 00000100:00000010:3.0:1773941420.813568:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85073800. 00000100:00000010:3.0:1773941420.813570:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885f2000. 00000100:00000010:3.0:1773941420.813572:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85070400. 00000100:00000010:3.0:1773941420.813574:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90246000. 00000100:00000010:3.0:1773941420.813578:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85071000. 00000100:00000010:3.0:1773941420.813579:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90242000. 00000100:00000010:3.0:1773941420.813582:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85072400. 00000100:00000010:3.0:1773941420.813585:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90240000. 00000100:00000010:3.0:1773941420.813590:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85072800. 00000100:00000010:3.0:1773941420.813594:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90244000. 00000100:00000010:3.0:1773941420.813599:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85072000. 00000100:00000010:3.0:1773941420.813603:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90254000. 00000100:00000010:3.0:1773941420.813606:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85073400. 00000100:00000010:3.0:1773941420.813608:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90256000. 00000100:00000010:3.0:1773941420.813619:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85070000. 00000100:00000010:3.0:1773941420.813622:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90252000. 00000100:00000010:3.0:1773941420.813625:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85073000. 00000100:00000010:3.0:1773941420.813627:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90250000. 00000100:00000010:3.0:1773941420.813630:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85070c00. 00000100:00000010:3.0:1773941420.813632:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889ce000. 00000100:00000010:3.0:1773941420.813648:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85072c00. 00000100:00000010:3.0:1773941420.813651:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889ca000. 00000100:00000010:3.0:1773941420.813654:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889ec400. 00000100:00000010:3.0:1773941420.813656:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889c8000. 00000100:00000010:3.0:1773941420.813659:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889ed000. 00000100:00000010:3.0:1773941420.813661:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889cc000. 00000100:00000010:3.0:1773941420.813663:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889ee400. 00000100:00000010:3.0:1773941420.813666:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d4e000. 00000100:00000010:3.0:1773941420.813668:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889ee800. 00000100:00000010:3.0:1773941420.813670:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d4a000. 00000100:00000010:3.0:1773941420.813673:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889ee000. 00000100:00000010:3.0:1773941420.813675:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d48000. 00000100:00000010:3.0:1773941420.813677:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889ef400. 00000100:00000010:3.0:1773941420.813680:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d4c000. 00000100:00000010:3.0:1773941420.813683:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889ec000. 00000100:00000010:3.0:1773941420.813685:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90df2000. 00000100:00000010:3.0:1773941420.813688:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889ef000. 00000100:00000010:3.0:1773941420.813690:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90df0000. 00000100:00000010:3.0:1773941420.813693:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889ecc00. 00000100:00000010:3.0:1773941420.813695:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90df4000. 00000100:00000010:3.0:1773941420.813697:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889eec00. 00000100:00000010:3.0:1773941420.813700:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90df6000. 00000100:00000010:3.0:1773941420.813702:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889edc00. 00000100:00000010:3.0:1773941420.813734:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d12000. 00000100:00000010:3.0:1773941420.813754:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889ed400. 00000100:00000010:3.0:1773941420.813758:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d10000. 00000100:00000010:3.0:1773941420.813762:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889efc00. 00000100:00000010:3.0:1773941420.813770:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d14000. 00000100:00000010:3.0:1773941420.813774:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889ec800. 00000100:00000010:3.0:1773941420.813776:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d16000. 00000100:00000010:3.0:1773941420.813779:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889ed800. 00000100:00000010:3.0:1773941420.813781:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885d2000. 00000100:00000010:3.0:1773941420.813784:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b889ef800. 00000100:00000010:3.0:1773941420.813787:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885d0000. 00000100:00000010:3.0:1773941420.813790:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0c29000. 00000100:00000010:3.0:1773941420.813794:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885d4000. 00000100:00000010:3.0:1773941420.813797:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0c2a400. 00000100:00000010:3.0:1773941420.813800:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885d6000. 00000100:00000010:3.0:1773941420.813803:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0c2a800. 00000100:00000010:3.0:1773941420.813805:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0ca4000. 00000100:00000010:3.0:1773941420.813809:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0c2a000. 00000100:00000010:3.0:1773941420.813811:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0ca6000. 00000100:00000010:3.0:1773941420.813814:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0c2b400. 00000100:00000010:3.0:1773941420.813817:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0ca2000. 00000100:00000010:3.0:1773941420.813820:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0c28000. 00000100:00000010:3.0:1773941420.813822:0:12100:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0ca0000. 00000100:00000010:3.0:1773941420.813825:0:12100:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0c2b000. 00000100:00000010:3.0:1773941420.813828:0:12100:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b90668800. 00000100:00000010:3.0:1773941420.813831:0:12100:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b8907b000. 00000100:00000010:3.0:1773941420.813834:0:12100:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b90f26800. 00000100:00000001:3.0:1773941420.813838:0:12100:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:3.0:1773941420.813841:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813844:0:12100:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941420.813846:0:12100:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941420.813848:0:12100:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff8b5b82c59740. 00000100:00000001:3.0:1773941420.813852:0:12100:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941420.813854:0:12100:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.813857:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813859:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b98115e40. 00000100:00000001:3.0:1773941420.813861:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813863:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813865:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b98115540. 00000100:00000001:3.0:1773941420.813874:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813876:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813877:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b98115d80. 00000100:00000001:3.0:1773941420.813879:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813881:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813882:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b91fc0780. 00000100:00000001:3.0:1773941420.813886:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813888:0:12100:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941420.813890:0:12100:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941420.813891:0:12100:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff8b5b82c59100. 00000100:00000001:3.0:1773941420.813895:0:12100:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941420.813896:0:12100:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.813898:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813899:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b91fc0e40. 00000100:00000001:3.0:1773941420.813901:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813902:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813903:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b91fc0900. 00000100:00000001:3.0:1773941420.813905:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813906:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813908:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b91fc0cc0. 00000100:00000001:3.0:1773941420.813909:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813911:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813912:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b850fa180. 00000100:00000010:3.0:1773941420.813914:0:12100:0:(nrs.c:1080:nrs_svcpt_cleanup_locked()) kfreed 'nrs': 112 at ffff8b5b90edb400. 00000100:00000001:3.0:1773941420.813918:0:12100:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000100:00000001:3.0:1773941420.813920:0:12100:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:3.0:1773941420.813921:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813922:0:12100:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941420.813923:0:12100:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941420.813925:0:12100:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff8b5b82c59300. 00000100:00000001:3.0:1773941420.813927:0:12100:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941420.813928:0:12100:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.813930:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813931:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b850fad80. 00000100:00000001:3.0:1773941420.813934:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813935:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813936:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b850faa80. 00000100:00000001:3.0:1773941420.813939:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813940:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813941:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b850fa3c0. 00000100:00000001:3.0:1773941420.813950:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813951:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813953:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b850facc0. 00000100:00000001:3.0:1773941420.813955:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813956:0:12100:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941420.813957:0:12100:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941420.813958:0:12100:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff8b5b82c59840. 00000100:00000001:3.0:1773941420.813961:0:12100:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941420.813962:0:12100:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.813965:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813966:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b850fac00. 00000100:00000001:3.0:1773941420.813968:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813969:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813971:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b850fa300. 00000100:00000001:3.0:1773941420.813972:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813973:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813975:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b850fa9c0. 00000100:00000001:3.0:1773941420.813977:0:12100:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941420.813978:0:12100:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941420.813980:0:12100:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b850fa000. 00000100:00000010:3.0:1773941420.813982:0:12100:0:(nrs.c:1080:nrs_svcpt_cleanup_locked()) kfreed 'nrs': 112 at ffff8b5b90edbe80. 00000100:00000001:3.0:1773941420.813984:0:12100:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000020:00000010:3.0:1773941420.814103:0:12100:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 3680 at ffff8b5b85e58000. 00000020:00000010:3.0:1773941420.814111:0:12100:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5ba0c28c00. 00000100:00000010:3.0:1773941420.814149:0:12100:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff8b5b850c0000. 00000100:00000010:3.0:1773941420.814159:0:12100:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff8b5b99127000. 00000100:00000010:3.0:1773941420.814164:0:12100:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff8b5b893c4000. 00000100:00000010:3.0:1773941420.814173:0:12100:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff8b5b99123000. 00000100:00000010:3.0:1773941420.814176:0:12100:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff8b5b88cb3000. 00000100:00000010:3.0:1773941420.814179:0:12100:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff8b5b885ad800. 00000100:00000010:3.0:1773941420.814182:0:12100:0:(service.c:3836:ptlrpc_service_free()) kfreed 'svc': 360 at ffff8b5b98182400. 00000100:00000001:3.0:1773941420.814186:0:12100:0:(service.c:3863:ptlrpc_unregister_service()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941420.814299:0:12030:0:(ldlm_lockd.c:142:have_expired_locks()) Process entered 00010000:00000001:0.0:1773941420.814303:0:12030:0:(ldlm_lockd.c:147:have_expired_locks()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941420.814321:0:12030:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068fc0. 00010000:00000010:3.0:1773941420.814331:0:12100:0:(ldlm_lockd.c:3586:ldlm_cleanup()) kfreed 'ldlm_state': 32 at ffff8b5b88570ca0. 00010000:00000001:3.0:1773941420.814337:0:12100:0:(ldlm_lockd.c:3589:ldlm_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941420.814337:0:12030:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b84d8a600. 00010000:00000001:3.0:1773941420.814342:0:12100:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00010000:00000010:3.0:1773941420.814347:0:12100:0:(ldlm_lib.c:647:client_obd_cleanup()) kfreed 'cli->cl_read_io_latency_by_size': 3920 at ffff8b5b90ee4000. 00000040:00000010:0.0:1773941420.814347:0:12030:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b84d8bc00. 00010000:00000010:3.0:1773941420.814355:0:12100:0:(ldlm_lib.c:651:client_obd_cleanup()) kfreed 'cli->cl_write_io_latency_by_size': 3920 at ffff8b5b90ee5000. 00000020:00000010:0.0:1773941420.814356:0:12030:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b9026a000. 00000020:00000010:0.0:1773941420.814362:0:12030:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b8849eeb0. 00010000:00000001:3.0:1773941420.814365:0:12100:0:(ldlm_lib.c:654:client_obd_cleanup()) Process leaving 10000000:00000010:3.0:1773941420.814369:0:12100:0:(mgc_request.c:834:mgc_device_free()) kfreed 'lu': 64 at ffff8b5b82c596c0. 00000020:00000010:0.0:1773941420.814370:0:12030:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b84d8a200. 10000000:00000001:3.0:1773941420.814373:0:12100:0:(mgc_request.c:836:mgc_device_free()) Process leaving (rc=0 : 0 : 0) 80000000:00000010:0.0:1773941420.814376:0:12030:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9d20. 00000020:00000010:3.0:1773941420.814378:0:12100:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b8809c200. 40000000:00000010:0.0:1773941420.814381:0:12030:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b85068380. 00000040:00000010:3.0:1773941420.814383:0:12100:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b8809c600. 00100000:00000010:0.0:1773941420.814387:0:12030:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b9026e000. 00000020:00000010:3.0:1773941420.814389:0:12100:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b98ac3000. 00040000:00000010:0.0:1773941420.814393:0:12030:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b884be000. 00000020:00000010:3.0:1773941420.814396:0:12100:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b8809da00. 00040000:00000010:0.0:1773941420.814398:0:12030:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b9026c800. 00040000:00000010:3.0:1773941420.814402:0:12100:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98a69900. 00040000:00000010:0.0:1773941420.814403:0:12030:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b84d8be00. 00040000:00000010:3.0:1773941420.814409:0:12100:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b8809d400. 00000004:00000010:0.0:1773941420.814409:0:12030:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b9026f800. 00000020:00000010:3.0:1773941420.814415:0:12100:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b8809c800. 00000004:00000010:0.0:1773941420.814416:0:12030:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5ba0b8a000. 00000020:00000001:3.0:1773941420.814423:0:12100:0:(obd_class.h:647:obd_cleanup()) Process leaving (rc=0 : 0 : 0) 00000004:00000010:0.0:1773941420.814424:0:12030:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b9026f000. 00000020:00000010:3.0:1773941420.814428:0:12100:0:(genops.c:66:obd_device_free()) slab-freed '(obd)': 6584 at ffff8b5b85514d28. 00002000:00000010:0.0:1773941420.814430:0:12030:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0b91400. 00000004:00000010:0.0:1773941420.814438:0:12030:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5ba0b92c00. 00000020:00000001:3.0:1773941420.814441:0:12100:0:(obd_config.c:923:class_detach()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941420.814442:0:12030:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84d8ae00. 00000020:00000001:3.0:1773941420.814445:0:12100:0:(obd_config.c:1655:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1773941420.814447:0:12030:0:(ldlm_lockd.c:290:expired_lock_main()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941420.814451:0:12100:0:(obd_config.c:2523:class_manual_cleanup()) kfreed 'lcfg': 72 at ffff8b5ba0bc9f00. 00000020:00000001:3.0:1773941420.814456:0:12100:0:(obd_config.c:2524:class_manual_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.814459:0:12100:0:(obd_mount.c:620:lustre_stop_mgc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.814462:0:12100:0:(obd_mount.c:720:lustre_put_lsi()) Process entered 00000020:01000004:3.0:1773941420.814464:0:12100:0:(obd_mount.c:724:lustre_put_lsi()) put ffff8b5b885e5800 1 00000020:00000001:3.0:1773941420.814466:0:12100:0:(obd_mount.c:679:lustre_free_lsi()) Process entered 00000020:01000004:3.0:1773941420.814467:0:12100:0:(obd_mount.c:682:lustre_free_lsi()) Freeing lsi ffff8b5b885e4800 00000020:00000010:3.0:1773941420.814471:0:12100:0:(obd_mount.c:631:lustre_put_lsm_free()) kfreed 'lmd->lmd_dev': 28 at ffff8b5b8441e000. 00000020:00000010:3.0:1773941420.814474:0:12100:0:(obd_mount.c:632:lustre_put_lsm_free()) kfreed 'lmd->lmd_profile': 14 at ffff8b5b84f3f430. 00000020:00000010:3.0:1773941420.814478:0:12100:0:(obd_mount.c:635:lustre_put_lsm_free()) kfreed 'lmd->lmd_opts': 17 at ffff8b5b8441e120. 00000020:00000010:3.0:1773941420.814482:0:12100:0:(obd_mount.c:642:lustre_put_lsm_free()) kfreed 'lmd->lmd_params': 4096 at ffff8b5b90ee2000. 00000020:00000010:3.0:1773941420.814486:0:12100:0:(obd_mount.c:644:lustre_put_lsm_free()) kfreed 'lmd': 128 at ffff8b5b90edb700. 00000020:00000010:3.0:1773941420.814489:0:12100:0:(obd_mount.c:691:lustre_free_lsi()) kfreed 'lsi': 1880 at ffff8b5b885e4800. 00000020:00000001:3.0:1773941420.814491:0:12100:0:(obd_mount.c:693:lustre_free_lsi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.814494:0:12100:0:(obd_mount.c:727:lustre_put_lsi()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1773941420.814496:0:12100:0:(obd_mount.c:929:lustre_common_put_super()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941420.814500:0:12100:0:(cl_object.c:885:cl_env_cache_purge()) Process entered 00000020:00000001:3.0:1773941420.814503:0:12100:0:(cl_object.c:903:cl_env_cache_purge()) Process leaving (rc=4294967295 : 4294967295 : ffffffff) 00000080:00000001:3.0:1773941420.814506:0:12100:0:(llite_lib.c:1639:ll_put_super()) Process leaving 00010000:00000010:0.1:1773941420.814781:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00010000:00000010:0.1:1773941420.814791:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962f00. 00000080:00000010:0.0:1773941420.876824:0:12130:0:(super25.c:253:lustre_init_fs_context()) kmalloced '(lmd)': 128 at ffff8b5b8848f700. 00000020:00000001:0.0:1773941420.876832:0:12130:0:(obd_mount.c:1366:lustre_parse_monolithic()) Process entered 00000020:00000010:0.0:1773941420.876836:0:12130:0:(obd_mount.c:1388:lustre_parse_monolithic()) kmalloced '(lmd->lmd_params)': 4096 at ffff8b5b88006000. 00000020:00000010:0.0:1773941420.876842:0:12130:0:(obd_mount.c:1096:lmd_parse_string()) kmalloced '(*handle)': 19 at ffff8b5b884b46a0. 00000020:00000010:0.0:1773941420.876846:0:12130:0:(obd_mount.c:1590:lustre_parse_monolithic()) kmalloced '(lmd->lmd_dev)': 48 at ffff8b5ba039cb00. 00000020:00000010:0.0:1773941420.876849:0:12130:0:(obd_mount.c:1646:lustre_parse_monolithic()) kmalloced '(lmd->lmd_profile)': 14 at ffff8b5b8859f3d0. 00000020:01000004:0.0:1773941420.876851:0:12130:0:(obd_mount.c:937:lmd_print()) mount data: 00000020:01000004:0.0:1773941420.876852:0:12130:0:(obd_mount.c:939:lmd_print()) profile: lustre-client 00000020:01000004:0.0:1773941420.876852:0:12130:0:(obd_mount.c:940:lmd_print()) device: 192.168.204.143@tcp,192.168.204.153@tcp:/lustre 00000020:00000001:0.0:1773941420.876854:0:12130:0:(obd_mount.c:1686:lustre_parse_monolithic()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941420.876937:0:12130:0:(super25.c:131:lustre_fill_super()) Process entered 00000080:01200004:0.0:1773941420.876938:0:12130:0:(super25.c:132:lustre_fill_super()) VFS Op: sb ffff8b5ba0b09800 00000020:00000001:0.0:1773941420.876940:0:12130:0:(obd_mount.c:652:lustre_init_lsi()) Process entered 00000020:00000010:0.0:1773941420.876942:0:12130:0:(obd_mount.c:656:lustre_init_lsi()) kmalloced '(lsi)': 1880 at ffff8b5ba0b0e000. 00000020:00000001:0.0:1773941420.876943:0:12130:0:(obd_mount.c:673:lustre_init_lsi()) Process leaving (rc=18446615824387072000 : -128249322479616 : ffff8b5ba0b0e000) 00000080:01000004:0.0:1773941420.876947:0:12130:0:(super25.c:169:lustre_fill_super()) Mounting client lustre-client 00000020:00000001:0.0:1773941420.876949:0:12130:0:(obd_mount.c:302:lustre_start_mgc()) Process entered 00000020:00000010:0.0:1773941420.876958:0:12130:0:(obd_mount.c:345:lustre_start_mgc()) kmalloced '(mgcname)': 23 at ffff8b5b83f0c460. 00000020:00000010:0.0:1773941420.876960:0:12130:0:(obd_mount.c:353:lustre_start_mgc()) kmalloced '(data)': 192 at ffff8b5ba0b6e000. 00000020:01000004:0.0:1773941420.876963:0:12130:0:(obd_mount.c:423:lustre_start_mgc()) Start MGC 'MGC192.168.204.143@tcp' 00000020:00000001:0.0:1773941420.876965:0:12130:0:(obd_mount.c:150:do_lcfg()) lcfg MGC192.168.204.143@tcp 0xcf005 192.168.204.143@tcp (null) (null) (null) 00000020:00000010:0.0:1773941420.876968:0:12130:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 88 at ffff8b5b85ef66c0. 00000020:00000080:0.0:1773941420.876969:0:12130:0:(obd_config.c:1506:class_process_config()) processing cmd: cf005 00000020:00000080:0.0:1773941420.876973:0:12130:0:(obd_config.c:1518:class_process_config()) adding mapping from uuid 192.168.204.143@tcp to nid 0x20000c0a8cc8f (192.168.204.143@tcp) 00000100:00000010:0.0:1773941420.876976:0:12130:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at ffff8b5ba0bf1c00. 00000100:00000040:0.0:1773941420.876979:0:12130:0:(lustre_peer.c:111:class_add_uuid()) found uuid 192.168.204.143@tcp 192.168.204.143@tcp cnt=1 00000400:00000200:0.0:1773941420.876981:0:12130:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00001000:0.0:1773941420.876983:0:12130:0:(api-ni.c:3210:LNetNIInit()) refs 1 00000400:00000200:0.0:1773941420.876985:0:12130:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.143@tcp 00000400:00000200:0.0:1773941420.876989:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941420.876992:0:12130:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.143@tcp NID flags 0x100001: -17 00000400:00000200:0.0:1773941420.876996:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941420.877000:0:12130:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941420.877002:0:12130:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp: 0. pending discovery 00000400:00000200:0.0:1773941420.877004:0:12130:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.143@tcp 00000100:00000010:0.0:1773941420.877005:0:12130:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at ffff8b5ba0bf1c00. 00000020:00000001:0.0:1773941420.877007:0:12130:0:(obd_config.c:1533:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1773941420.877009:0:12130:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 88 at ffff8b5b85ef66c0. 00000020:00000001:0.0:1773941420.877011:0:12130:0:(obd_mount.c:150:do_lcfg()) lcfg MGC192.168.204.143@tcp 0xcf005 192.168.204.143@tcp (null) (null) (null) 00000020:00000010:0.0:1773941420.877013:0:12130:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 88 at ffff8b5b85ef66c0. 00000020:00000080:0.0:1773941420.877014:0:12130:0:(obd_config.c:1506:class_process_config()) processing cmd: cf005 00000020:00000080:0.0:1773941420.877016:0:12130:0:(obd_config.c:1518:class_process_config()) adding mapping from uuid 192.168.204.143@tcp to nid 0x20000c0a8cc99 (192.168.204.153@tcp) 00000100:00000010:0.0:1773941420.877017:0:12130:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at ffff8b5ba0bf1c00. 00000100:00000040:0.0:1773941420.877018:0:12130:0:(lustre_peer.c:111:class_add_uuid()) found uuid 192.168.204.143@tcp 192.168.204.153@tcp cnt=2 00000400:00000200:0.0:1773941420.877022:0:12130:0:(peer.c:1295:LNetAddPeer()) Add peer with 2 NIDs 00000400:00001000:0.0:1773941420.877023:0:12130:0:(api-ni.c:3210:LNetNIInit()) refs 1 00000400:00000200:0.0:1773941420.877024:0:12130:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.143@tcp 00000400:00000200:0.0:1773941420.877026:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941420.877028:0:12130:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.143@tcp NID flags 0x100001: -17 00000400:00000200:0.0:1773941420.877029:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941420.877031:0:12130:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941420.877032:0:12130:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp: 0. pending discovery 00000400:00000200:0.0:1773941420.877033:0:12130:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.143@tcp 00000400:00000200:0.0:1773941420.877034:0:12130:0:(peer.c:1309:LNetAddPeer()) Adding NID[1]: 192.168.204.153@tcp 00000400:00000200:0.0:1773941420.877035:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000010:0.0:1773941420.877038:0:12130:0:(peer.c:229:lnet_peer_alloc()) alloc '(lp)': 376 at ffff8b5b84fee600. 00000400:00000200:0.0:1773941420.877040:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:0.0:1773941420.877041:0:12130:0:(peer.c:262:lnet_peer_alloc()) ffff8b5b84fee600 nid 192.168.204.153@tcp 00000400:00000010:0.0:1773941420.877043:0:12130:0:(peer.c:193:lnet_peer_net_alloc()) alloc '(lpn)': 72 at ffff8b5b85ef6420. 00000400:00000200:0.0:1773941420.877044:0:12130:0:(peer.c:202:lnet_peer_net_alloc()) ffff8b5b85ef6420 net tcp 00000400:00000200:0.0:1773941420.877046:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000010:0.0:1773941420.877047:0:12130:0:(peer.c:137:lnet_peer_ni_alloc()) alloc '(lpni)': 384 at ffff8b5b84fef800. 00000400:00000200:0.0:1773941420.877050:0:12130:0:(peer.c:183:lnet_peer_ni_alloc()) ffff8b5b84fef800 nid 192.168.204.153@tcp 00000400:00000200:0.0:1773941420.877053:0:12130:0:(peer.c:1091:lnet_peer_ni_clr_non_mr_pref_nid()) peer 192.168.204.153@tcp: -2 00000400:00000200:0.0:1773941420.877056:0:12130:0:(peer.c:1639:lnet_peer_attach_peer_ni()) peer 192.168.204.153@tcp NID 192.168.204.153@tcp flags 0x1 00000400:00000200:0.0:1773941420.877058:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:0.0:1773941420.877060:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:0.0:1773941420.877070:0:12130:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.153@tcp: 0 00000400:00000200:0.0:1773941420.877071:0:12130:0:(peer.c:2727:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:0.0:1773941420.877072:0:12130:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941420.877074:0:12130:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.153@tcp NID 192.168.204.153@tcp: 0. pending discovery 00000100:00000010:0.0:1773941420.877076:0:12130:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at ffff8b5ba0bf1c00. 00000020:00000001:0.0:1773941420.877077:0:12130:0:(obd_config.c:1533:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1773941420.877078:0:12130:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 88 at ffff8b5b85ef66c0. 00000020:00000010:0.0:1773941420.877080:0:12130:0:(obd_mount.c:487:lustre_start_mgc()) kmalloced '(uuid)': 40 at ffff8b5ba039c7c0. 00000020:01000004:0.0:1773941420.877084:0:12130:0:(obd_mount.c:208:lustre_start_simple()) Starting OBD MGC192.168.204.143@tcp (typ=mgc) 00000020:00000001:0.0:1773941420.877086:0:12130:0:(obd_mount.c:150:do_lcfg()) lcfg MGC192.168.204.143@tcp 0xcf001 mgc 1080e6b5-6a6c-4762-b0ec-c785990fec17 (null) (null) 00000020:00000010:0.0:1773941420.877088:0:12130:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 120 at ffff8b5b8848fa00. 00000020:00000080:0.0:1773941420.877090:0:12130:0:(obd_config.c:1506:class_process_config()) processing cmd: cf001 00000020:00000001:0.0:1773941420.877094:0:12130:0:(obd_config.c:700:class_attach()) Process entered 00000400:00000200:1.0:1773941420.877095:0:2350:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000020:00000001:0.0:1773941420.877097:0:12130:0:(genops.c:349:class_newdev()) Process entered 00000400:00000200:1.0:1773941420.877104:0:2350:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.153@tcp(ffff8b5b84fee600) state 0x41 00000020:00000010:0.0:1773941420.877105:0:12130:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at ffff8b5b982fb370. 00000020:00000080:0.0:1773941420.877109:0:12130:0:(genops.c:412:class_newdev()) Allocate new device MGC192.168.204.143@tcp (ffff8b5b982fb370) 00000020:00000001:0.0:1773941420.877111:0:12130:0:(genops.c:950:__class_new_export()) Process entered 00000400:00000010:1.0:1773941420.877112:0:2350:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at ffff8b5b902a1400. 00000020:00000010:0.0:1773941420.877114:0:12130:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5ba0b0c000. 00000020:00000001:0.0:1773941420.877117:0:12130:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941420.877119:0:12130:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5ba0b0c000 with handle 0xa33a5be14eae1c3d to hash 00000400:00000010:1.0:1773941420.877120:0:2350:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1088. 00000020:00000001:0.0:1773941420.877121:0:12130:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00000020:00000001:0.0:1773941420.877125:0:12130:0:(obd_class.h:1008:obd_init_export()) Process entered 00000020:00000001:0.0:1773941420.877127:0:12130:0:(obd_class.h:1012:obd_init_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941420.877129:0:12130:0:(genops.c:1020:__class_new_export()) Process leaving (rc=18446615824387063808 : -128249322487808 : ffff8b5ba0b0c000) 00000400:00000010:1.0:1773941420.877130:0:2350:0:(lib-lnet.h:478:lnet_rspt_alloc()) alloc 'rspt': 56 at ffff8b5b817f4ea8. 00000020:00000040:0.0:1773941420.877131:0:12130:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0c000 : new refcount 1 00000020:00000040:0.0:1773941420.877137:0:12130:0:(obd_config.c:1030:class_incref()) incref MGC192.168.204.143@tcp (ffff8b5b982fb370) now 2 - obd_device_list 00000400:00000200:1.0:1773941420.877140:0:2350:0:(lib-move.c:5547:LNetGet()) LNetGet -> 12345-192.168.204.153@tcp 00000020:00000001:0.0:1773941420.877142:0:12130:0:(genops.c:554:class_register_device()) Process leaving (rc=0 : 0 : 0) 00000020:00000080:0.0:1773941420.877144:0:12130:0:(obd_config.c:757:class_attach()) OBD: dev 0 attached type mgc with refcount 2 00000020:00000001:0.0:1773941420.877146:0:12130:0:(obd_config.c:759:class_attach()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941420.877147:0:12130:0:(obd_config.c:1512:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1773941420.877149:0:12130:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 120 at ffff8b5b8848fa00. 00000400:00000200:1.0:1773941420.877152:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000020:00000001:0.0:1773941420.877153:0:12130:0:(obd_mount.c:150:do_lcfg()) lcfg MGC192.168.204.143@tcp 0xcf003 MGS 192.168.204.143@tcp (null) (null) 00000020:00000010:0.0:1773941420.877158:0:12130:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 104 at ffff8b5b8848fa00. 00000020:00000080:0.0:1773941420.877160:0:12130:0:(obd_config.c:1506:class_process_config()) processing cmd: cf003 00000400:00000200:1.0:1773941420.877163:0:2350:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.153@tcp local destination 00000020:00000001:0.0:1773941420.877163:0:12130:0:(obd_config.c:779:class_setup()) Process entered 00000400:00000200:1.0:1773941420.877170:0:2350:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941420.877175:0:2350:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000001:00000001:0.0:1773941420.877175:0:12130:0:(hash.c:1038:cfs_hash_create()) Process entered 00000001:00000010:0.0:1773941420.877178:0:12130:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at ffff8b5b99101e00. 00000400:00000200:1.0:1773941420.877179:0:2350:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000001:00000010:0.0:1773941420.877182:0:12130:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff8b5b83f0c9c0. 00000400:00000200:1.0:1773941420.877185:0:2350:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000001:00000010:0.0:1773941420.877186:0:12130:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5b84feec00. 00000001:00000010:0.0:1773941420.877189:0:12130:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5b84fee200. 00000001:00000010:0.0:1773941420.877192:0:12130:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5b84feea00. 00000400:00000200:1.0:1773941420.877193:0:2350:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [106:106:256:4294967295] 192.168.204.153@tcp LPNI seq info [1:1:8:4294967295] 00000001:00000010:0.0:1773941420.877195:0:12130:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff8b5b84fefa00. 00000020:00000001:0.0:1773941420.877198:0:12130:0:(obd_class.h:557:obd_setup()) Process entered 00000020:00000010:0.0:1773941420.877202:0:12130:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84fef600. 00000400:00000200:1.0:1773941420.877203:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000020:00000010:0.0:1773941420.877206:0:12130:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff8b5b8848f600. 00000400:00000200:1.0:1773941420.877211:0:2350:0:(lib-move.c:2009:lnet_handle_send()) rspt_next_hop_nid = 192.168.204.153@tcp 00000020:00000010:0.0:1773941420.877211:0:12130:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff8b5b85eeb190. 00008000:00000010:0.0:1773941420.877216:0:12130:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8b5b902a3328. 00000400:00000200:1.0:1773941420.877221:0:2350:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.153@tcp:192.168.204.153@tcp) : GET try# 0 00000020:00000010:0.0:1773941420.877224:0:12130:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84fee000. 00000020:00000010:0.0:1773941420.877227:0:12130:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b84fee400. 00000800:00000200:1.0:1773941420.877229:0:2350:0:(socklnd_cb.c:1001:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.204.153@tcp 00000040:00000010:0.0:1773941420.877230:0:12130:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b84feee00. 00000020:00000010:0.0:1773941420.877236:0:12130:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5ba0b0a800. 00000800:00000010:1.0:1773941420.877237:0:2350:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at ffff8b5b9024b900. 00000020:00000010:0.0:1773941420.877239:0:12130:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b84fee800. 00040000:00000010:0.0:1773941420.877244:0:12130:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b99101000. 00040000:00000010:0.0:1773941420.877247:0:12130:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b84fef200. 00000400:00000200:1.0:1773941420.877248:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 10000000:00000001:0.0:1773941420.877251:0:12130:0:(mgc_request.c:850:mgc_device_alloc()) Process entered 00000800:00000010:1.0:1773941420.877255:0:2350:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at ffff8b5b885de6c0. 10000000:00000010:0.0:1773941420.877255:0:12130:0:(mgc_request.c:852:mgc_device_alloc()) kmalloced '(lu)': 64 at ffff8b5ba039c580. 00000100:00000001:0.0:1773941420.877258:0:12130:0:(ptlrpcd.c:908:ptlrpcd_addref()) Process entered 00000100:00000001:0.0:1773941420.877260:0:12130:0:(ptlrpcd.c:917:ptlrpcd_addref()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1773941420.877262:0:2350:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at ffff8b5b885dee40. 00010000:00000001:0.0:1773941420.877263:0:12130:0:(ldlm_lib.c:298:client_obd_setup()) Process entered 00000800:00000200:1.0:1773941420.877280:0:2350:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (2) 00010000:00000010:0.0:1773941420.877282:0:12130:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at ffff8b5b98acd000. 00010000:00000010:0.0:1773941420.877288:0:12130:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at ffff8b5b98acc000. 00010000:00000001:0.0:1773941420.877291:0:12130:0:(ldlm_lockd.c:3030:ldlm_get_ref()) Process entered 00000800:00000200:1.1:1773941420.877292:0:2350:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (2) 00010000:00000001:0.0:1773941420.877294:0:12130:0:(ldlm_lockd.c:3351:ldlm_setup()) Process entered 00010000:00000010:0.0:1773941420.877297:0:12130:0:(ldlm_lockd.c:3356:ldlm_setup()) kmalloced '(ldlm_state)': 32 at ffff8b5b83f0c2e0. 00000400:00000200:1.0:1773941420.877309:0:2350:0:(peer.c:3788:lnet_peer_send_ping()) peer 192.168.204.153@tcp 00000400:00000200:1.0:1773941420.877319:0:2350:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.153@tcp(ffff8b5b84fee600) state 0x241 rc 0 00010000:00000001:0.0:1773941420.877328:0:12130:0:(ldlm_resource.c:68:ldlm_debugfs_setup()) Process entered 00010000:00000001:0.0:1773941420.877357:0:12130:0:(ldlm_resource.c:78:ldlm_debugfs_setup()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877360:0:12130:0:(service.c:701:ptlrpc_register_service()) Process entered 00000100:00000010:0.0:1773941420.877363:0:12130:0:(service.c:745:ptlrpc_register_service()) kmalloced '(service)': 360 at ffff8b5b91fc7000. 00000100:00000010:0.0:1773941420.877366:0:12130:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff8b5ba0bf1400. 00000100:00000010:0.0:1773941420.877377:0:12130:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff8b5b90d2c000. 00000100:00000010:0.0:1773941420.877383:0:12130:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff8b5b98acf000. 00000100:00000010:0.0:1773941420.877387:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0bf3400. 00000100:00000010:0.0:1773941420.877391:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b850de000. 00000100:00100000:0.0:1773941420.877393:0:12130:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/1 left), rc = 0 00000100:00000010:0.0:1773941420.877396:0:12130:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff8b5ba0bf1800. 00000100:00000010:0.0:1773941420.877405:0:12130:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff8b5b98a98000. 00000100:00000010:0.0:1773941420.877409:0:12130:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff8b5b98ac8000. 00000100:00000010:0.0:1773941420.877413:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0bf1000. 00000100:00000010:0.0:1773941420.877416:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b850dc000. 00000100:00100000:0.0:1773941420.877418:0:12130:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/1 left), rc = 0 00000400:00000200:0.0:1773941420.877424:0:12130:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 15 lazy 00000020:00000010:0.0:1773941420.877450:0:12130:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5ba0bf3c00. 00000020:00000010:0.0:1773941420.877455:0:12130:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at ffff8b5b98acb000. 00000100:00000001:0.0:1773941420.877492:0:12130:0:(nrs.c:1006:nrs_svcpt_setup_locked()) Process entered 00000100:00000001:0.0:1773941420.877495:0:12130:0:(nrs.c:924:nrs_register_policies_locked()) Process entered 00000100:00000001:0.0:1773941420.877497:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.877500:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5ba0b6e0c0. 00000100:00000001:0.0:1773941420.877503:0:12130:0:(nrs.c:211:nrs_policy_start_locked()) Process entered 00000100:00000010:0.0:1773941420.877507:0:12130:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff8b5ba039ce00. 00000100:00000001:0.0:1773941420.877509:0:12130:0:(nrs.c:336:nrs_policy_start_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877511:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877513:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.877516:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5ba0b6ef00. 00000100:00000001:0.0:1773941420.877518:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877520:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.877522:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5ba0b6e6c0. 00000100:00000001:0.0:1773941420.877523:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877525:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.877526:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5ba0b6ee40. 00000100:00000001:0.0:1773941420.877528:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877529:0:12130:0:(nrs.c:944:nrs_register_policies_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877530:0:12130:0:(nrs.c:985:nrs_svcpt_setup_locked0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877531:0:12130:0:(nrs.c:1022:nrs_svcpt_setup_locked()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941420.877533:0:12130:0:(nrs.c:1034:nrs_svcpt_setup_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877534:0:12130:0:(nrs.c:1006:nrs_svcpt_setup_locked()) Process entered 00000100:00000001:0.0:1773941420.877535:0:12130:0:(nrs.c:924:nrs_register_policies_locked()) Process entered 00000100:00000001:0.0:1773941420.877535:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.877537:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b83f1f840. 00000100:00000001:0.0:1773941420.877538:0:12130:0:(nrs.c:211:nrs_policy_start_locked()) Process entered 00000100:00000010:0.0:1773941420.877539:0:12130:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff8b5ba039c8c0. 00000100:00000001:0.0:1773941420.877540:0:12130:0:(nrs.c:336:nrs_policy_start_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877541:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877542:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.877543:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b83f1fa80. 00000100:00000001:0.0:1773941420.877544:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877544:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.877546:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b845cf300. 00000100:00000001:0.0:1773941420.877552:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877553:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.877555:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b845cf900. 00000100:00000001:0.0:1773941420.877556:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877557:0:12130:0:(nrs.c:944:nrs_register_policies_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877558:0:12130:0:(nrs.c:985:nrs_svcpt_setup_locked0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877559:0:12130:0:(nrs.c:1022:nrs_svcpt_setup_locked()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941420.877560:0:12130:0:(nrs.c:1034:nrs_svcpt_setup_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.877585:0:12130:0:(nrs.c:1379:ptlrpc_service_nrs_setup()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1773941420.877587:0:12130:0:(service.c:848:ptlrpc_register_service()) ldlm_cbd: Started, listening on portal 15 00000100:00000001:0.0:1773941420.877588:0:12130:0:(service.c:3400:ptlrpc_start_threads()) Process entered 00000100:00000001:0.0:1773941420.877589:0:12130:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:0.0:1773941420.877591:0:12130:0:(service.c:3441:ptlrpc_start_thread()) ldlm_cbd[0] started 0 min 2 max 34 00000100:00000010:0.0:1773941420.877594:0:12130:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b91fe9200. 00000100:00100000:0.0:1773941420.877596:0:12130:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cb00_000' 00000100:00000001:0.0:1773941420.877862:0:12131:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:0.0:1773941420.877884:0:12131:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5ba039c000. 00000020:00000010:0.0:1773941420.877890:0:12131:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5ba039c240. 00000020:00000010:0.0:1773941420.877897:0:12131:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b8a0f2400. 00000020:00000010:0.0:1773941420.877902:0:12131:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b88909c00. 00000040:00000010:0.0:1773941420.877905:0:12131:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b88908600. 00000020:00000010:0.0:1773941420.877919:0:12131:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5ba0b0b000. 00000020:00000010:0.0:1773941420.877923:0:12131:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b98298f18. 00000020:00000010:0.0:1773941420.877926:0:12131:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b88908c00. 80000000:00000010:0.0:1773941420.877931:0:12131:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b85ef6000. 40000000:00000010:0.0:1773941420.877934:0:12131:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5ba039cf40. 00100000:00000010:0.0:1773941420.877940:0:12131:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b8a324800. 00040000:00000010:0.0:1773941420.877943:0:12131:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b99101300. 00040000:00000010:0.0:1773941420.877948:0:12131:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b8a327000. 00040000:00000010:0.0:1773941420.877950:0:12131:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b88908400. 00000004:00000010:0.0:1773941420.877955:0:12131:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b86885800. 00000004:00000010:0.0:1773941420.877961:0:12131:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b98aca000. 00000004:00000010:0.0:1773941420.877966:0:12131:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b90db1800. 00002000:00000010:0.0:1773941420.877970:0:12131:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0bf0000. 00000004:00000010:0.0:1773941420.877976:0:12131:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5b98f0e000. 00000100:00000200:0.0:1773941420.877987:0:12131:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_cbd: registering portal 15 00000400:00000010:0.0:1773941420.877993:0:12131:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93c08. 00000400:00000010:0.0:1773941420.877999:0:12131:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b88ea7cc0. 00000100:00000010:0.0:1773941420.878006:0:12131:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b90db0000. 00000100:00000200:0.0:1773941420.878022:0:12131:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000001:2.0:1773941420.878043:0:12130:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.878048:0:12130:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:2.0:1773941420.878049:0:12130:0:(service.c:3441:ptlrpc_start_thread()) ldlm_cbd[0] started 1 min 2 max 34 00000100:00000010:2.0:1773941420.878053:0:12130:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b89317600. 00000100:00100000:2.0:1773941420.878056:0:12130:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cb00_001' 00000100:00000001:0.0:1773941420.878231:0:12132:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:0.0:1773941420.878246:0:12132:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5ba039c780. 00000020:00000010:0.0:1773941420.878250:0:12132:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5ba039c040. 00000020:00000010:0.0:1773941420.878254:0:12132:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b88909400. 00000020:00000010:0.0:1773941420.878257:0:12132:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b88908000. 00000040:00000010:0.0:1773941420.878259:0:12132:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b88909a00. 00000020:00000010:0.0:1773941420.878263:0:12132:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90db0800. 00000020:00000010:0.0:1773941420.878280:0:12132:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b9829e4a0. 00000020:00000010:0.0:1773941420.878283:0:12132:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b88909800. 80000000:00000010:0.0:1773941420.878286:0:12132:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b85ef6300. 40000000:00000010:0.0:1773941420.878288:0:12132:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5ba039c080. 00100000:00000010:0.0:1773941420.878291:0:12132:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b90db5800. 00040000:00000010:0.0:1773941420.878294:0:12132:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b99101500. 00040000:00000010:0.0:1773941420.878297:0:12132:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b90db7000. 00040000:00000010:0.0:1773941420.878299:0:12132:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b88908a00. 00000004:00000010:0.0:1773941420.878303:0:12132:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b90db3000. 00000004:00000010:0.0:1773941420.878308:0:12132:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b885ba000. 00000004:00000010:0.0:1773941420.878311:0:12132:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b90db7800. 00002000:00000010:0.0:1773941420.878314:0:12132:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b98f0d800. 00000004:00000010:0.0:1773941420.878317:0:12132:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5b98f0c000. 00000100:00000010:0.0:1773941420.878320:0:12132:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b90db4800. 00000100:00000200:0.0:1773941420.878340:0:12132:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000001:2.0:1773941420.878351:0:12130:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.878356:0:12130:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:2.0:1773941420.878364:0:12130:0:(service.c:3441:ptlrpc_start_thread()) ldlm_cbd[1] started 0 min 2 max 34 00000100:00000010:2.0:1773941420.878369:0:12130:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b89316a00. 00000100:00100000:2.0:1773941420.878372:0:12130:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cb01_000' 00000100:00000001:3.0:1773941420.878987:0:12133:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:3.0:1773941420.879003:0:12133:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5b85068140. 00000020:00000010:3.0:1773941420.879013:0:12133:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068400. 00000800:00000010:2.0:1773941420.879023:0:2340:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4712 at ffff8b5ba0c74000. 00000800:00000010:2.0:1773941420.879029:0:2340:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at ffff8b5ba0b540c0. 00000800:00000010:2.0:1773941420.879034:0:2340:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at ffff8b5b845ad1c0. 00000800:00000010:2.0:1773941420.879049:0:2340:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at ffff8b5b845ad1c0. 00000020:00000010:3.0:1773941420.879085:0:12133:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b8809c800. 00000020:00000010:3.0:1773941420.879087:0:12133:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b8809ca00. 00000040:00000010:3.0:1773941420.879089:0:12133:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b8809ce00. 00000020:00000010:3.0:1773941420.879092:0:12133:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b98ac4000. 00000020:00000010:3.0:1773941420.879095:0:12133:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b88499420. 00000020:00000010:3.0:1773941420.879097:0:12133:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b8809c600. 80000000:00000010:3.0:1773941420.879099:0:12133:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9180. 40000000:00000010:3.0:1773941420.879101:0:12133:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b85068740. 00100000:00000010:3.0:1773941420.879103:0:12133:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b98ac2800. 00040000:00000010:3.0:1773941420.879105:0:12133:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98a69900. 00040000:00000010:3.0:1773941420.879107:0:12133:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b98ac3000. 00040000:00000010:3.0:1773941420.879109:0:12133:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b83991e00. 00000004:00000010:3.0:1773941420.879111:0:12133:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b98ac7000. 00000004:00000010:3.0:1773941420.879123:0:12133:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5ba0b88000. 00000004:00000010:3.0:1773941420.879127:0:12133:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b98ac0800. 00002000:00000010:3.0:1773941420.879131:0:12133:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b90262c00. 00000004:00000010:3.0:1773941420.879138:0:12133:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5b90263c00. 00000100:00000200:3.0:1773941420.879141:0:12133:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_cbd: registering portal 15 00000400:00000010:3.0:1773941420.879146:0:12133:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87e18. 00000400:00000010:3.0:1773941420.879150:0:12133:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90ec5540. 00000100:00000010:3.0:1773941420.879155:0:12133:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b98ac0000. 00000100:00000200:3.0:1773941420.879166:0:12133:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000001:0.0:1773941420.879187:0:12130:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.879191:0:12130:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:0.0:1773941420.879192:0:12130:0:(service.c:3441:ptlrpc_start_thread()) ldlm_cbd[1] started 1 min 2 max 34 00000100:00000010:0.0:1773941420.879198:0:12130:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b88909e00. 00000100:00100000:0.0:1773941420.879202:0:12130:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cb01_001' 00000100:00000001:0.0:1773941420.879405:0:12134:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:2.0:1773941420.879459:0:12134:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5b845ad1c0. 00000020:00000010:2.0:1773941420.879463:0:12134:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b845ada40. 00000020:00000010:2.0:1773941420.879466:0:12134:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b89317e00. 00000020:00000010:2.0:1773941420.879469:0:12134:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b89316400. 00000040:00000010:2.0:1773941420.879471:0:12134:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b89317a00. 00000020:00000010:2.0:1773941420.879473:0:12134:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90f23800. 00000020:00000010:2.0:1773941420.879476:0:12134:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b88558000. 00000020:00000010:2.0:1773941420.879478:0:12134:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b89316000. 80000000:00000010:2.0:1773941420.879480:0:12134:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0166240. 40000000:00000010:2.0:1773941420.879482:0:12134:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b845ad180. 00100000:00000010:2.0:1773941420.879484:0:12134:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b90f25000. 00040000:00000010:2.0:1773941420.879486:0:12134:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98bb0400. 00040000:00000010:2.0:1773941420.879488:0:12134:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b90f24800. 00040000:00000010:2.0:1773941420.879489:0:12134:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b89316600. 00000004:00000010:2.0:1773941420.879491:0:12134:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b90f24000. 00000004:00000010:2.0:1773941420.879494:0:12134:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b88905000. 00000004:00000010:2.0:1773941420.879496:0:12134:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b90f27800. 00002000:00000010:2.0:1773941420.879498:0:12134:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b83f09000. 00000004:00000010:2.0:1773941420.879500:0:12134:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5b83f09c00. 00000100:00000010:2.0:1773941420.879502:0:12134:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b90f26000. 00000100:00000200:2.0:1773941420.879512:0:12134:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000001:0.0:1773941420.879522:0:12130:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.879526:0:12130:0:(service.c:3418:ptlrpc_start_threads()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.879533:0:12130:0:(service.c:857:ptlrpc_register_service()) Process leaving (rc=18446615824140365824 : -128249569185792 : ffff8b5b91fc7000) 00000100:00000001:0.0:1773941420.879536:0:12130:0:(service.c:701:ptlrpc_register_service()) Process entered 00000100:00000010:0.0:1773941420.879539:0:12130:0:(service.c:745:ptlrpc_register_service()) kmalloced '(service)': 360 at ffff8b5b845c4200. 00000100:00000010:0.0:1773941420.879543:0:12130:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff8b5b98f0c800. 00000100:00000010:0.0:1773941420.879552:0:12130:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff8b5ba0c4c000. 00000100:00000010:0.0:1773941420.879557:0:12130:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff8b5b885bd000. 00000100:00000010:0.0:1773941420.879561:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b98f0f000. 00000100:00000010:0.0:1773941420.879565:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b850d8000. 00000100:00000010:0.0:1773941420.879567:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b98f0dc00. 00000100:00000010:0.0:1773941420.879570:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b850da000. 00000100:00000010:0.0:1773941420.879572:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b98f0fc00. 00000100:00000010:0.0:1773941420.879576:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b72000. 00000100:00000010:0.0:1773941420.879579:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b84c97400. 00000100:00000010:0.0:1773941420.879582:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b76000. 00000100:00000010:0.0:1773941420.879585:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b84c96000. 00000100:00000010:0.0:1773941420.879588:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b74000. 00000100:00000010:0.0:1773941420.879590:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b84c95800. 00000100:00000010:0.0:1773941420.879593:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b70000. 00000100:00000010:0.0:1773941420.879594:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b84c96400. 00000100:00000010:0.0:1773941420.879598:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b8856a000. 00000100:00000010:0.0:1773941420.879600:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b84c95000. 00000100:00000010:0.0:1773941420.879603:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b8856e000. 00000100:00000010:0.0:1773941420.879604:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b84c94400. 00000100:00000010:0.0:1773941420.879607:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b8856c000. 00000100:00000010:0.0:1773941420.879608:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b84c97800. 00000100:00000010:0.0:1773941420.879611:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88568000. 00000100:00000010:0.0:1773941420.879612:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b84c96800. 00000100:00000010:0.0:1773941420.879615:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88560000. 00000100:00000010:0.0:1773941420.879617:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b84c97000. 00000100:00000010:0.0:1773941420.879620:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88562000. 00000100:00000010:0.0:1773941420.879621:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b84c94c00. 00000100:00000010:0.0:1773941420.879627:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88566000. 00000100:00000010:0.0:1773941420.879629:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b84c96c00. 00000100:00000010:0.0:1773941420.879631:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88564000. 00000100:00000010:0.0:1773941420.879633:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b84c95c00. 00000100:00000010:0.0:1773941420.879636:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885ce000. 00000100:00000010:0.0:1773941420.879638:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b84c95400. 00000100:00000010:0.0:1773941420.879641:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885cc000. 00000100:00000010:0.0:1773941420.879643:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b84c94800. 00000100:00000010:0.0:1773941420.879645:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885c8000. 00000100:00000010:0.0:1773941420.879648:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8902ac00. 00000100:00000010:0.0:1773941420.879650:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885ca000. 00000100:00000010:0.0:1773941420.879652:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0dc4800. 00000100:00000010:0.0:1773941420.879655:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885c4000. 00000100:00000010:0.0:1773941420.879657:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0dc5c00. 00000100:00000010:0.0:1773941420.879660:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885c0000. 00000100:00000010:0.0:1773941420.879661:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0dc5400. 00000100:00000010:0.0:1773941420.879664:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885c2000. 00000100:00000010:0.0:1773941420.879665:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0dc5800. 00000100:00000010:0.0:1773941420.879668:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885c6000. 00000100:00000010:0.0:1773941420.879670:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88cb3000. 00000100:00000010:0.0:1773941420.879675:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b850b8000. 00000100:00000010:0.0:1773941420.879678:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88cb3800. 00000100:00000010:0.0:1773941420.879682:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b850ba000. 00000100:00000010:0.0:1773941420.879685:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88cb1c00. 00000100:00000010:0.0:1773941420.879689:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b850be000. 00000100:00000010:0.0:1773941420.879693:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8800f800. 00000100:00000010:0.0:1773941420.879699:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b850bc000. 00000100:00000010:0.0:1773941420.879724:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8800d400. 00000100:00000010:0.0:1773941420.879731:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b981a6000. 00000100:00000010:0.0:1773941420.879735:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8800e000. 00000100:00000010:0.0:1773941420.879739:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b981a2000. 00000100:00000010:0.0:1773941420.879746:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8800d800. 00000100:00000010:0.0:1773941420.879751:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b981a0000. 00000100:00000010:0.0:1773941420.879753:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8800c000. 00000100:00000010:0.0:1773941420.879759:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b78000. 00000100:00000010:0.0:1773941420.879762:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8800f000. 00000100:00000010:0.0:1773941420.879766:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b7e000. 00000100:00000010:0.0:1773941420.879770:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8800dc00. 00000100:00000010:0.0:1773941420.879774:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b7c000. 00000100:00000010:0.0:1773941420.879778:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8800cc00. 00000100:00000010:0.0:1773941420.879783:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b83d0a000. 00000100:00000010:0.0:1773941420.879786:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8800c400. 00000400:00000200:2.0:1773941420.879790:0:2340:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000100:00000010:0.0:1773941420.879795:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b829e4000. 00000800:00000200:2.1:1773941420.879797:0:2340:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000100:00000010:0.0:1773941420.879799:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8800fc00. 00000100:00000010:0.0:1773941420.879806:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b829e2000. 00000800:00000200:2.1:1773941420.879809:0:2340:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000100:00000010:0.0:1773941420.879810:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8800ec00. 00000800:00000200:2.1:1773941420.879817:0:2340:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b900 type 2, nob 96 niov 1 nkiov 0 00000100:00000010:0.0:1773941420.879817:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b91ca8000. 00000100:00000010:0.0:1773941420.879822:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8800e400. 00000800:00000200:2.0:1773941420.879827:0:2340:0:(socklnd.c:1183:ksocknal_create_conn()) New conn 12345-192.168.204.153@tcp p 3.x 192.168.204.43 -> 192.168.204.153:988 incarnation:1773939896378632739 sched[1] 00000100:00000010:0.0:1773941420.879828:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b91cac000. 00000800:00000010:2.0:1773941420.879834:0:2340:0:(socklnd.c:1191:ksocknal_create_conn()) kfreed 'hello': 144 at ffff8b5ba0b540c0. 00000100:00000010:0.0:1773941420.879835:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0b92c00. 00000100:00000010:0.0:1773941420.879842:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b832aa000. 00000100:00000010:0.0:1773941420.879846:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0b91400. 00000100:00000010:0.0:1773941420.879850:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b832a8000. 00000100:00000010:0.0:1773941420.879853:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0b91000. 00000100:00000010:0.0:1773941420.879857:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b832ac000. 00000100:00000010:0.0:1773941420.879861:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0b93c00. 00000800:00000001:3.0:1773941420.879873:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000100:00000010:0.0:1773941420.879873:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b8857a000. 00000800:00000001:3.0:1773941420.879879:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1773941420.879879:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0b92800. 00000100:00000010:0.0:1773941420.879884:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88578000. 00000100:00000010:0.0:1773941420.879889:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0b91800. 00000100:00000010:0.0:1773941420.879894:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b8857c000. 00000100:00000010:0.0:1773941420.879897:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0b90c00. 00000400:00000200:2.0:1773941420.879909:0:2340:0:(lib-socket.c:453:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.204.153:988 00000100:00000010:0.0:1773941420.879909:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b8857e000. 00000100:00000010:0.0:1773941420.879914:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0b90400. 00000800:00000200:3.0:1773941420.879915:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941420.879920:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at ffff8b5b9024b900. 00000400:00000200:3.0:1773941420.879924:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000010:0.0:1773941420.879927:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90278000. 00000400:00000200:3.0:1773941420.879929:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: GET: OK 00000100:00000010:0.0:1773941420.879933:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0b92000. 00000400:00000200:3.0:1773941420.879935:0:2347:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:3.0:1773941420.879939:0:2347:0:(peer.c:3033:lnet_discovery_event_send()) Ping Send to 192.168.204.153@tcp: 0 00000100:00000010:0.0:1773941420.879941:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b9027c000. 00000100:00000010:0.0:1773941420.879946:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0b93800. 00000100:00000010:0.0:1773941420.879952:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b9027e000. 00000100:00000010:0.0:1773941420.879956:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0b92400. 00000100:00000010:0.0:1773941420.879962:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b9027a000. 00000100:00000010:0.0:1773941420.879965:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0b91c00. 00000100:00000010:0.0:1773941420.879973:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889da000. 00000100:00000010:0.0:1773941420.879976:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0b90000. 00000100:00000010:0.0:1773941420.879979:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889d8000. 00000100:00000010:0.0:1773941420.879981:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0b90800. 00000100:00000010:0.0:1773941420.879984:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889dc000. 00000100:00000010:0.0:1773941420.879987:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91f8ec00. 00000100:00000010:0.0:1773941420.879990:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889de000. 00000100:00000010:0.0:1773941420.879992:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91f8d400. 00000100:00000010:0.0:1773941420.879998:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88988000. 00000100:00000010:0.0:1773941420.880000:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91f8cc00. 00000100:00000010:0.0:1773941420.880003:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b8898c000. 00000100:00000010:0.0:1773941420.880004:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91f8e800. 00000100:00000010:0.0:1773941420.880007:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b8898e000. 00000100:00000010:0.0:1773941420.880008:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91f8f400. 00000100:00000010:0.0:1773941420.880011:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b8898a000. 00000100:00000010:0.0:1773941420.880012:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91f8e400. 00000100:00000010:0.0:1773941420.880018:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0cd6000. 00000100:00000010:0.0:1773941420.880021:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88011800. 00000100:00000010:0.0:1773941420.880024:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0cd2000. 00000100:00000010:0.0:1773941420.880025:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88012400. 00000100:00000010:0.0:1773941420.880028:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0cd0000. 00000100:00000010:0.0:1773941420.880032:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88010800. 00000100:00000010:0.0:1773941420.880035:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0cd4000. 00000100:00000010:0.0:1773941420.880037:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88011000. 00000100:00000010:0.0:1773941420.880043:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0c00000. 00000100:00000010:0.0:1773941420.880045:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88012800. 00000100:00000010:0.0:1773941420.880048:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0c04000. 00000100:00000010:0.0:1773941420.880049:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88011c00. 00000100:00000010:0.0:1773941420.880052:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0c06000. 00000100:00000010:0.0:1773941420.880053:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b88012000. 00000100:00000010:0.0:1773941420.880056:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0c02000. 00000100:00100000:0.0:1773941420.880058:0:12130:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_canceld: allocate 64 new 8192-byte reqbufs (0/64 left), rc = 0 00000100:00000010:0.0:1773941420.880060:0:12130:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff8b5b88011400. 00000100:00000010:0.0:1773941420.880068:0:12130:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff8b5ba0718000. 00000100:00000010:0.0:1773941420.880073:0:12130:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff8b5b885b9000. 00000100:00000010:0.0:1773941420.880076:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91c4d400. 00000100:00000010:0.0:1773941420.880082:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90df0000. 00000100:00000010:0.0:1773941420.880084:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91c4d000. 00000100:00000010:0.0:1773941420.880086:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90df4000. 00000100:00000010:0.0:1773941420.880088:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91c4fc00. 00000100:00000010:0.0:1773941420.880090:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90df6000. 00000100:00000010:0.0:1773941420.880092:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91e4a400. 00000100:00000010:0.0:1773941420.880095:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90df2000. 00000100:00000010:0.0:1773941420.880097:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b91e49000. 00000100:00000010:0.0:1773941420.880103:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d4e000. 00000100:00000010:0.0:1773941420.880105:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b890c3000. 00000100:00000010:0.0:1773941420.880109:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d4a000. 00000100:00000010:0.0:1773941420.880110:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b890c0800. 00000100:00000010:0.0:1773941420.880113:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d48000. 00000100:00000010:0.0:1773941420.880114:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b890c2000. 00000100:00000010:0.0:1773941420.880117:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d4c000. 00000100:00000010:0.0:1773941420.880119:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b99ef4000. 00000100:00000010:0.0:1773941420.880127:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889ca000. 00000100:00000010:0.0:1773941420.880129:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b99ef7400. 00000100:00000010:0.0:1773941420.880132:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889c8000. 00000100:00000010:0.0:1773941420.880133:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b99ef4800. 00000100:00000010:0.0:1773941420.880136:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889cc000. 00000100:00000010:0.0:1773941420.880138:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f3a400. 00000100:00000010:0.0:1773941420.880141:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889ce000. 00000100:00000010:0.0:1773941420.880142:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f3a800. 00000100:00000010:0.0:1773941420.880149:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90256000. 00000100:00000010:0.0:1773941420.880151:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f3ac00. 00000100:00000010:0.0:1773941420.880155:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90252000. 00000100:00000010:0.0:1773941420.880156:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f38400. 00000100:00000010:0.0:1773941420.880160:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90250000. 00000100:00000010:0.0:1773941420.880161:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f38800. 00000100:00000010:0.0:1773941420.880165:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90254000. 00000100:00000010:0.0:1773941420.880166:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f3bc00. 00000100:00000010:0.0:1773941420.880174:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90242000. 00000100:00000010:0.0:1773941420.880175:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f39800. 00000100:00000010:0.0:1773941420.880179:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90240000. 00000100:00000010:0.0:1773941420.880180:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f39000. 00000100:00000010:0.0:1773941420.880184:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90244000. 00000100:00000010:0.0:1773941420.880185:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b90f3b800. 00000100:00000010:0.0:1773941420.880189:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90246000. 00000100:00000010:0.0:1773941420.880191:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b99f56800. 00000100:00000010:0.0:1773941420.880197:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885f4000. 00000100:00000010:0.0:1773941420.880199:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b99f57800. 00000100:00000010:0.0:1773941420.880202:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885f6000. 00000100:00000010:0.0:1773941420.880203:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b99f54400. 00000100:00000010:0.0:1773941420.880206:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885f2000. 00000100:00000010:0.0:1773941420.880208:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b99f55000. 00000100:00000010:0.0:1773941420.880210:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b885f0000. 00000100:00000010:0.0:1773941420.880216:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5bc2072000. 00000100:00000010:0.0:1773941420.880223:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b9022e000. 00000100:00000010:0.0:1773941420.880226:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5bc2073400. 00000100:00000010:0.0:1773941420.880229:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b9022a000. 00000100:00000010:0.0:1773941420.880231:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8a3a2c00. 00000100:00000010:0.0:1773941420.880235:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90228000. 00000100:00000010:0.0:1773941420.880237:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5bc2076000. 00000100:00000010:0.0:1773941420.880240:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b9022c000. 00000100:00000010:0.0:1773941420.880242:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5bc2068000. 00000100:00000010:0.0:1773941420.880248:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b8505a000. 00000100:00000010:0.0:1773941420.880250:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5bc206b000. 00000100:00000010:0.0:1773941420.880253:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b85058000. 00000100:00000010:0.0:1773941420.880255:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5bc2069c00. 00000100:00000010:0.0:1773941420.880257:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b8505c000. 00000100:00000010:0.0:1773941420.880259:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5bc2069400. 00000100:00000010:0.0:1773941420.880262:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b8505e000. 00000100:00000010:0.0:1773941420.880275:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85611000. 00000100:00000010:0.0:1773941420.880284:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889f0000. 00000100:00000010:0.0:1773941420.880286:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85610c00. 00000100:00000010:0.0:1773941420.880289:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889f4000. 00000100:00000010:0.0:1773941420.880291:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b85613800. 00000100:00000010:0.0:1773941420.880294:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889f6000. 00000100:00000010:0.0:1773941420.880295:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b8801c800. 00000100:00000010:0.0:1773941420.880298:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b889f2000. 00000100:00000010:0.0:1773941420.880304:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0bacc00. 00000100:00000010:0.0:1773941420.880311:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88992000. 00000100:00000010:0.0:1773941420.880314:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0baec00. 00000100:00000010:0.0:1773941420.880317:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88990000. 00000100:00000010:0.0:1773941420.880319:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0badc00. 00000100:00000010:0.0:1773941420.880322:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88994000. 00000100:00000010:0.0:1773941420.880323:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0bad400. 00000100:00000010:0.0:1773941420.880330:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88996000. 00000100:00000010:0.0:1773941420.880333:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0bafc00. 00000100:00000010:0.0:1773941420.880339:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a3e000. 00000100:00000010:0.0:1773941420.880342:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0bac800. 00000100:00000010:0.0:1773941420.880345:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a3a000. 00000100:00000010:0.0:1773941420.880347:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0bad800. 00000100:00000010:0.0:1773941420.880351:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a38000. 00000100:00000010:0.0:1773941420.880353:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0baf800. 00000100:00000010:0.0:1773941420.880357:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a3c000. 00000100:00000010:0.0:1773941420.880358:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0bac400. 00000100:00000010:0.0:1773941420.880365:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d6a000. 00000100:00000010:0.0:1773941420.880368:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0bad000. 00000100:00000010:0.0:1773941420.880371:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d68000. 00000100:00000010:0.0:1773941420.880373:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0bae400. 00000100:00000010:0.0:1773941420.880376:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d6c000. 00000100:00000010:0.0:1773941420.880378:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0bae800. 00000800:00000010:2.0:1773941420.880379:0:2340:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4712 at ffff8b5ba0c76000. 00000100:00000010:0.0:1773941420.880382:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90d6e000. 00000100:00000010:0.0:1773941420.880383:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0bae000. 00000800:00000010:2.0:1773941420.880390:0:2340:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at ffff8b5ba0b540c0. 00000100:00000010:0.0:1773941420.880392:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a12000. 00000800:00000010:2.0:1773941420.880395:0:2340:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at ffff8b5b845ad100. 00000100:00000010:0.0:1773941420.880396:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0baf400. 00000100:00000010:0.0:1773941420.880402:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a10000. 00000100:00000010:0.0:1773941420.880406:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0bac000. 00000800:00000010:2.0:1773941420.880407:0:2340:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at ffff8b5b845ad100. 00000100:00000010:0.0:1773941420.880410:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a14000. 00000100:00000010:0.0:1773941420.880414:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5ba0baf000. 00000100:00000010:0.0:1773941420.880417:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b98a16000. 00000100:00000010:0.0:1773941420.880424:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b850c2c00. 00000100:00000010:0.0:1773941420.880430:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b18000. 00000100:00000010:0.0:1773941420.880432:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b850c1c00. 00000100:00000010:0.0:1773941420.880434:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b1c000. 00000100:00000010:0.0:1773941420.880435:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b850c1400. 00000100:00000010:0.0:1773941420.880438:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b1e000. 00000100:00000010:0.0:1773941420.880444:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b850c3c00. 00000100:00000010:0.0:1773941420.880448:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5ba0b1a000. 00000100:00000010:0.0:1773941420.880451:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b850c0800. 00000100:00000010:0.0:1773941420.880460:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90da4000. 00000100:00000010:0.0:1773941420.880463:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b850c1800. 00000100:00000010:0.0:1773941420.880467:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90da6000. 00000100:00000010:0.0:1773941420.880469:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b850c3800. 00000100:00000010:0.0:1773941420.880473:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90da2000. 00000100:00000010:0.0:1773941420.880475:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b850c0400. 00000100:00000010:0.0:1773941420.880479:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b90da0000. 00000100:00000010:0.0:1773941420.880482:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b850c1000. 00000100:00000010:0.0:1773941420.880491:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88944000. 00000100:00000010:0.0:1773941420.880495:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b850c2400. 00000100:00000010:0.0:1773941420.880498:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88946000. 00000100:00000010:0.0:1773941420.880501:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b850c2800. 00000100:00000010:0.0:1773941420.880505:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88942000. 00000100:00000010:0.0:1773941420.880508:0:12130:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff8b5b850c2000. 00000100:00000010:0.0:1773941420.880512:0:12130:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff8b5b88940000. 00000100:00100000:0.0:1773941420.880514:0:12130:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_canceld: allocate 64 new 8192-byte reqbufs (0/64 left), rc = 0 00000400:00000200:0.0:1773941420.880518:0:12130:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 17 lazy 00000020:00000010:0.0:1773941420.880565:0:12130:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5b850c3400. 00000020:00000010:0.0:1773941420.880568:0:12130:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at ffff8b5b885bb000. 00000100:00000001:0.0:1773941420.880599:0:12130:0:(nrs.c:1006:nrs_svcpt_setup_locked()) Process entered 00000100:00000001:0.0:1773941420.880600:0:12130:0:(nrs.c:924:nrs_register_policies_locked()) Process entered 00000100:00000001:0.0:1773941420.880602:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880605:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b88ea7480. 00000100:00000001:0.0:1773941420.880607:0:12130:0:(nrs.c:211:nrs_policy_start_locked()) Process entered 00000100:00000010:0.0:1773941420.880609:0:12130:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff8b5ba039c1c0. 00000100:00000001:0.0:1773941420.880610:0:12130:0:(nrs.c:336:nrs_policy_start_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880612:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880613:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880615:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b88ea7a80. 00000100:00000001:0.0:1773941420.880616:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880622:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880623:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b88ea7f00. 00000100:00000001:0.0:1773941420.880624:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880625:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880627:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b88ea7780. 00000100:00000001:0.0:1773941420.880628:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880629:0:12130:0:(nrs.c:944:nrs_register_policies_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880630:0:12130:0:(nrs.c:985:nrs_svcpt_setup_locked0()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941420.880632:0:12130:0:(nrs.c:1026:nrs_svcpt_setup_locked()) kmalloced '(svcpt->scp_nrs_hp)': 112 at ffff8b5b8848fc80. 00000100:00000001:0.0:1773941420.880634:0:12130:0:(nrs.c:924:nrs_register_policies_locked()) Process entered 00000100:00000001:0.0:1773941420.880634:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880635:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b88ea7900. 00000100:00000001:0.0:1773941420.880636:0:12130:0:(nrs.c:211:nrs_policy_start_locked()) Process entered 00000100:00000010:0.0:1773941420.880637:0:12130:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff8b5ba039c100. 00000100:00000001:0.0:1773941420.880638:0:12130:0:(nrs.c:336:nrs_policy_start_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880639:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880640:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880641:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b88ea76c0. 00000100:00000001:0.0:1773941420.880642:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880643:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880643:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b88ea7b40. 00000100:00000001:0.0:1773941420.880644:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880645:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880646:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b88ea7e40. 00000100:00000001:0.0:1773941420.880647:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880648:0:12130:0:(nrs.c:944:nrs_register_policies_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880649:0:12130:0:(nrs.c:985:nrs_svcpt_setup_locked0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880650:0:12130:0:(nrs.c:1034:nrs_svcpt_setup_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880651:0:12130:0:(nrs.c:1006:nrs_svcpt_setup_locked()) Process entered 00000100:00000001:0.0:1773941420.880651:0:12130:0:(nrs.c:924:nrs_register_policies_locked()) Process entered 00000100:00000001:0.0:1773941420.880652:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880653:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b88ea7180. 00000100:00000001:0.0:1773941420.880654:0:12130:0:(nrs.c:211:nrs_policy_start_locked()) Process entered 00000100:00000010:0.0:1773941420.880655:0:12130:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff8b5ba039ce80. 00000100:00000001:0.0:1773941420.880655:0:12130:0:(nrs.c:336:nrs_policy_start_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880656:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880657:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880661:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b88ea73c0. 00000100:00000001:0.0:1773941420.880662:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880663:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880664:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b88ea7000. 00000100:00000001:0.0:1773941420.880665:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880666:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880667:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b88ea7840. 00000100:00000001:0.0:1773941420.880668:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880669:0:12130:0:(nrs.c:944:nrs_register_policies_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880670:0:12130:0:(nrs.c:985:nrs_svcpt_setup_locked0()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941420.880671:0:12130:0:(nrs.c:1026:nrs_svcpt_setup_locked()) kmalloced '(svcpt->scp_nrs_hp)': 112 at ffff8b5b8848f000. 00000100:00000001:0.0:1773941420.880672:0:12130:0:(nrs.c:924:nrs_register_policies_locked()) Process entered 00000100:00000001:0.0:1773941420.880673:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880676:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b87df9900. 00000100:00000001:0.0:1773941420.880678:0:12130:0:(nrs.c:211:nrs_policy_start_locked()) Process entered 00000100:00000010:0.0:1773941420.880680:0:12130:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff8b5ba039c3c0. 00000100:00000001:0.0:1773941420.880681:0:12130:0:(nrs.c:336:nrs_policy_start_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880683:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880684:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880687:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b89064780. 00000100:00000001:0.0:1773941420.880689:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880690:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880693:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b89064c00. 00000100:00000001:0.0:1773941420.880695:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880697:0:12130:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941420.880700:0:12130:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff8b5b89064a80. 00000100:00000001:0.0:1773941420.880701:0:12130:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880722:0:12130:0:(nrs.c:944:nrs_register_policies_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880724:0:12130:0:(nrs.c:985:nrs_svcpt_setup_locked0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941420.880725:0:12130:0:(nrs.c:1034:nrs_svcpt_setup_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1773941420.880740:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.880744:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941420.880746:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.880754:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941420.880754:0:12130:0:(nrs.c:1379:ptlrpc_service_nrs_setup()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1773941420.880757:0:12130:0:(service.c:848:ptlrpc_register_service()) ldlm_canceld: Started, listening on portal 17 00000400:00000200:3.0:1773941420.880760:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000100:00000001:0.0:1773941420.880761:0:12130:0:(service.c:3400:ptlrpc_start_threads()) Process entered 00000100:00000001:0.0:1773941420.880763:0:12130:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:0.0:1773941420.880766:0:12130:0:(service.c:3441:ptlrpc_start_thread()) ldlm_canceld[0] started 0 min 3 max 34 00000400:00000200:3.0:1773941420.880768:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : REPLY - for me 00000100:00000010:0.0:1773941420.880770:0:12130:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b845c4600. 00000100:00100000:0.0:1773941420.880773:0:12130:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cn00_000' 00000400:00000200:3.0:1773941420.880774:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941420.880822:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941420.880832:0:2347:0:(lib-move.c:4647:lnet_parse_reply()) 192.168.204.43@tcp: Reply from 12345-192.168.204.153@tcp of length 48/48 into md 0x5d1 00000800:00000001:3.0:1773941420.880838:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.880845:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941420.880848:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941420.880851:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: REPLY: OK 00000400:00000200:3.0:1773941420.880856:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1088 00000400:00000010:3.0:1773941420.880858:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1088. 00000400:00000200:3.0:1773941420.880861:0:2347:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 3 00000400:00000200:3.0:1773941420.880864:0:2347:0:(peer.c:2911:lnet_discovery_event_reply()) Peer 192.168.204.153@tcp has discovery enabled 00000400:00000200:3.0:1773941420.880866:0:2347:0:(peer.c:2923:lnet_discovery_event_reply()) peer 192.168.204.153@tcp(ffff8b5b84fee600) is MR 00000400:00000200:3.0:1773941420.880869:0:2347:0:(peer.c:2992:lnet_discovery_event_reply()) peer 192.168.204.153@tcp data present 363512030. state = 0x241 00000400:00000200:3.0:1773941420.880872:0:2347:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.153@tcp: -114 00000800:00000001:3.0:1773941420.880885:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.880889:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941420.880894:0:2350:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1773941420.880899:0:2350:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.153@tcp(ffff8b5b84fee600) state 0xc1 00000400:00000200:0.0:1773941420.880903:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941420.880909:0:2350:0:(peer.c:432:lnet_peer_del_locked()) peer 192.168.204.143@tcp 00000400:00000200:0.0:1773941420.880913:0:2350:0:(peer.c:354:lnet_peer_detach_peer_ni_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp 00000400:00000200:0.0:1773941420.880916:0:2350:0:(peer.c:2187:lnet_destroy_peer_ni_locked()) ffff8b5b90e7f800 nid 192.168.204.143@tcp 00000400:00000010:0.0:1773941420.880918:0:2350:0:(peer.c:2217:lnet_destroy_peer_ni_locked()) kfreed 'lpni': 384 at ffff8b5b90e7f800. 00000400:00000200:0.0:1773941420.880921:0:2350:0:(peer.c:212:lnet_destroy_peer_net_locked()) ffff8b5b85ef6ba0 net tcp 00000400:00000010:0.0:1773941420.880923:0:2350:0:(peer.c:219:lnet_destroy_peer_net_locked()) kfreed 'lpn': 72 at ffff8b5b85ef6ba0. 00000400:00000200:0.0:1773941420.880926:0:2350:0:(peer.c:270:lnet_destroy_peer_locked()) ffff8b5b90e7fe00 nid 192.168.204.143@tcp 00000400:00000010:0.0:1773941420.880928:0:2350:0:(peer.c:300:lnet_destroy_peer_locked()) kfreed 'lp': 376 at ffff8b5b90e7fe00. 00000400:00000200:0.0:1773941420.880930:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941420.880933:0:2350:0:(peer.c:137:lnet_peer_ni_alloc()) alloc '(lpni)': 384 at ffff8b5b845c5c00. 00000400:00000200:0.0:1773941420.880936:0:2350:0:(peer.c:183:lnet_peer_ni_alloc()) ffff8b5b845c5c00 nid 192.168.204.143@tcp 00000400:00000200:0.0:1773941420.880940:0:2350:0:(peer.c:1639:lnet_peer_attach_peer_ni()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp flags 0x11 00000400:00000200:0.0:1773941420.880943:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:0.0:1773941420.880945:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:0.0:1773941420.880947:0:2350:0:(peer.c:1899:lnet_peer_add_nid()) peer 192.168.204.143@tcp NID 192.168.204.153@tcp flags 0x11: 0 00000400:00000010:0.0:1773941420.880950:0:2350:0:(peer.c:3265:lnet_peer_merge_data()) alloc '(curnis)': 40 at ffff8b5ba039cf00. 00000400:00000010:0.0:1773941420.880952:0:2350:0:(peer.c:3266:lnet_peer_merge_data()) alloc '(addnis)': 48 at ffff8b5ba039c640. 00000400:00000010:0.0:1773941420.880954:0:2350:0:(peer.c:3267:lnet_peer_merge_data()) alloc '(delnis)': 40 at ffff8b5ba039c9c0. 00000400:00000200:0.0:1773941420.880956:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:0.0:1773941420.880960:0:2350:0:(peer.c:566:lnet_peer_del_nid()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp flags 0x11: -1 00000400:00020000:0.0:1773941420.880964:0:2350:0:(peer.c:3365:lnet_peer_merge_data()) Error deleting NID 192.168.204.143@tcp from peer 192.168.204.143@tcp: -1 00000100:00000001:2.0:1773941420.880987:0:12135:0:(service.c:2998:ptlrpc_main()) Process entered 00000400:00000200:2.0:1773941420.881155:0:2340:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000800:00000200:2.1:1773941420.881158:0:2340:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:2.0:1773941420.881164:0:2340:0:(socklnd.c:1183:ksocknal_create_conn()) New conn 12345-192.168.204.153@tcp p 3.x 192.168.204.43 -> 192.168.204.153:988 incarnation:1773939896378632739 sched[1] 00000800:00000010:2.0:1773941420.881168:0:2340:0:(socklnd.c:1191:ksocknal_create_conn()) kfreed 'hello': 144 at ffff8b5ba0b540c0. 00000800:00000001:3.0:1773941420.881191:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941420.881195:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941420.881205:0:2340:0:(lib-socket.c:453:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.204.153:988 00000400:00000200:2.0:1773941420.881232:0:2340:0:(lib-socket.c:453:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.204.153:988 00000800:00000010:2.0:1773941420.881693:0:2340:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4712 at ffff8b5ba0c72000. 00000800:00000010:2.0:1773941420.881696:0:2340:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at ffff8b5ba0b540c0. 00000800:00000010:2.0:1773941420.881698:0:2340:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at ffff8b5b845ad100. 00000800:00000010:2.0:1773941420.881727:0:2340:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at ffff8b5b845ad100. 00000100:00000010:1.0:1773941420.881777:0:12135:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5b850b0900. 00000020:00000010:1.0:1773941420.881783:0:12135:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b850b0740. 00000020:00000010:1.0:1773941420.881788:0:12135:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b902a1800. 00000020:00000010:1.0:1773941420.881793:0:12135:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b902a0800. 00000020:00000010:1.0:1773941420.881797:0:12135:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5b880f9a80. 00000040:00000010:1.0:1773941420.881801:0:12135:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b902a1200. 00000020:00000010:1.0:1773941420.881805:0:12135:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b8896b800. 00000020:00000010:1.0:1773941420.881809:0:12135:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b98a04670. 00000020:00000010:1.0:1773941420.881813:0:12135:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b902a0c00. 80000000:00000010:1.0:1773941420.881816:0:12135:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f9ae0. 40000000:00000010:1.0:1773941420.881819:0:12135:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b850b0140. 00008000:00000010:1.0:1773941420.881824:0:12135:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d26a60. 00100000:00000010:1.0:1773941420.881828:0:12135:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b88969800. 00040000:00000010:1.0:1773941420.881831:0:12135:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b9024bc00. 00040000:00000010:1.0:1773941420.881835:0:12135:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b88968800. 00040000:00000010:1.0:1773941420.881839:0:12135:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b902a0600. 00000004:00000010:1.0:1773941420.881843:0:12135:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b88969000. 00000004:00000010:1.0:1773941420.881847:0:12135:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b90f35000. 00000004:00000010:1.0:1773941420.881851:0:12135:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b8896c000. 00002000:00000010:1.0:1773941420.881855:0:12135:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0734400. 00000004:00000010:1.0:1773941420.881859:0:12135:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5ba0735000. 00000100:00000200:1.0:1773941420.881863:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881868:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034a50. 00000400:00000010:1.0:1773941420.881872:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b885decc0. 00000100:00000200:1.0:1773941420.881877:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881879:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034478. 00000400:00000010:1.0:1773941420.881881:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b885def00. 00000100:00000200:1.0:1773941420.881883:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881886:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b850341b8. 00000400:00000010:1.0:1773941420.881888:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b885de180. 00000100:00000200:1.0:1773941420.881891:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881892:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034318. 00000400:00000010:1.0:1773941420.881895:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b885ded80. 00000100:00000200:1.0:1773941420.881906:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881909:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034528. 00000400:00000010:1.0:1773941420.881911:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b885de0c0. 00000100:00000200:1.0:1773941420.881914:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881916:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034160. 00000400:00000010:1.0:1773941420.881918:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b885de480. 00000100:00000200:1.0:1773941420.881920:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881923:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034f78. 00000400:00000010:1.0:1773941420.881925:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b885de3c0. 00000100:00000200:1.0:1773941420.881927:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881929:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034420. 00000400:00000010:1.0:1773941420.881932:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b885de540. 00000100:00000200:1.0:1773941420.881933:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881935:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b850342c0. 00000400:00000010:1.0:1773941420.881938:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b885de240. 00000100:00000200:1.0:1773941420.881940:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881942:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b850340b0. 00000400:00000010:1.0:1773941420.881944:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b885de000. 00000100:00000200:1.0:1773941420.881946:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881949:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034688. 00000400:00000010:1.0:1773941420.881951:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b885deb40. 00000100:00000200:1.0:1773941420.881953:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881956:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b850344d0. 00000400:00000010:1.0:1773941420.881959:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85028f00. 00000100:00000200:1.0:1773941420.881962:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881964:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b850346e0. 00000400:00000010:1.0:1773941420.881967:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85028240. 00000100:00000200:1.0:1773941420.881968:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881971:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034058. 00000400:00000010:1.0:1773941420.881973:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85028780. 00000100:00000200:1.0:1773941420.881975:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881978:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034e18. 00000400:00000010:1.0:1773941420.881981:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b850289c0. 00000100:00000200:1.0:1773941420.881983:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.881986:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034aa8. 00000400:00000010:1.0:1773941420.881994:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85028000. 00000100:00000200:1.0:1773941420.881997:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882000:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034c60. 00000400:00000010:1.0:1773941420.882003:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85028300. 00000100:00000200:1.0:1773941420.882006:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882008:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034210. 00000400:00000010:1.0:1773941420.882010:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85028600. 00000100:00000200:1.0:1773941420.882012:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882014:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b850343c8. 00000400:00000010:1.0:1773941420.882016:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b850280c0. 00000100:00000200:1.0:1773941420.882018:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882020:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b850349a0. 00000400:00000010:1.0:1773941420.882022:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85028c00. 00000100:00000200:1.0:1773941420.882024:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882026:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b850347e8. 00000400:00000010:1.0:1773941420.882029:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85028d80. 00000100:00000200:1.0:1773941420.882030:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882032:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034ec8. 00000400:00000010:1.0:1773941420.882034:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85028480. 00000100:00000200:1.0:1773941420.882036:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882039:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034d68. 00000400:00000010:1.0:1773941420.882043:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b91c56d80. 00000100:00000200:1.0:1773941420.882045:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882047:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034c08. 00000400:00000010:1.0:1773941420.882050:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b903fda80. 00000100:00000200:1.0:1773941420.882052:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882054:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034dc0. 00000400:00000010:1.0:1773941420.882057:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b903fdc00. 00000100:00000200:1.0:1773941420.882059:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882061:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034840. 00000400:00000010:1.0:1773941420.882064:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b903fd300. 00000100:00000200:1.0:1773941420.882066:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882069:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b850348f0. 00000400:00000010:1.0:1773941420.882071:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b903fd900. 00000100:00000200:1.0:1773941420.882073:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882081:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034000. 00000400:00000010:1.0:1773941420.882084:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b903fdb40. 00000100:00000200:1.0:1773941420.882087:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882088:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b850345d8. 00000400:00000010:1.0:1773941420.882091:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b903fd240. 00000100:00000200:1.0:1773941420.882093:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882095:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034b58. 00000400:00000010:1.0:1773941420.882097:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b903fde40. 00000100:00000200:1.0:1773941420.882099:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882101:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034f20. 00000400:00000010:1.0:1773941420.882104:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b903fd000. 00000100:00000200:1.0:1773941420.882106:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882108:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034580. 00000400:00000010:1.0:1773941420.882110:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b903fd540. 00000100:00000200:1.0:1773941420.882112:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882114:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034898. 00000400:00000010:1.0:1773941420.882116:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b903fd3c0. 00000100:00000200:1.0:1773941420.882118:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882121:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034e70. 00000400:00000010:1.0:1773941420.882123:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b903fd600. 00000100:00000200:1.0:1773941420.882125:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882127:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034370. 00000400:00000010:1.0:1773941420.882130:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b852999c0. 00000100:00000200:1.0:1773941420.882131:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882133:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034268. 00000400:00000010:1.0:1773941420.882135:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85299cc0. 00000100:00000200:1.0:1773941420.882137:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882139:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034bb0. 00000400:00000010:1.0:1773941420.882141:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b852996c0. 00000100:00000200:1.0:1773941420.882143:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882145:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034cb8. 00000400:00000010:1.0:1773941420.882147:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85299600. 00000100:00000200:1.0:1773941420.882149:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882151:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034738. 00000400:00000010:1.0:1773941420.882154:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0d99c00. 00000100:00000200:1.0:1773941420.882161:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882163:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034790. 00000400:00000010:1.0:1773941420.882167:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfa900. 00000100:00000200:1.0:1773941420.882170:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882172:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034108. 00000400:00000010:1.0:1773941420.882174:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfab40. 00000100:00000200:1.0:1773941420.882177:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882178:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034630. 00000400:00000010:1.0:1773941420.882181:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfa9c0. 00000100:00000200:1.0:1773941420.882183:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882184:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034b00. 00000400:00000010:1.0:1773941420.882187:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfa540. 00000100:00000200:1.0:1773941420.882188:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882190:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034948. 00000400:00000010:1.0:1773941420.882192:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfaa80. 00000100:00000200:1.0:1773941420.882194:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882196:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85034d10. 00000400:00000010:1.0:1773941420.882198:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfa3c0. 00000100:00000200:1.0:1773941420.882200:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882201:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b850349f8. 00000400:00000010:1.0:1773941420.882203:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfac00. 00000100:00000200:1.0:1773941420.882205:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882215:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441adc0. 00000400:00000010:1.0:1773941420.882219:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfa840. 00000100:00000200:1.0:1773941420.882221:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882223:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a840. 00000400:00000010:1.0:1773941420.882226:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfa240. 00000100:00000200:1.0:1773941420.882227:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882229:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a8f0. 00000400:00000010:1.0:1773941420.882231:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfa480. 00000100:00000200:1.0:1773941420.882234:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882235:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a000. 00000400:00000010:1.0:1773941420.882237:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfa0c0. 00000100:00000200:1.0:1773941420.882239:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882241:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a5d8. 00000400:00000010:1.0:1773941420.882247:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfa300. 00000100:00000200:1.0:1773941420.882249:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882251:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441ab58. 00000400:00000010:1.0:1773941420.882253:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfa600. 00000100:00000200:1.0:1773941420.882255:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882257:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441af20. 00000400:00000010:1.0:1773941420.882260:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfad80. 00000100:00000200:1.0:1773941420.882262:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882263:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a580. 00000400:00000010:1.0:1773941420.882279:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfae40. 00000100:00000200:1.0:1773941420.882282:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882284:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a898. 00000400:00000010:1.0:1773941420.882286:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfacc0. 00000100:00000200:1.0:1773941420.882289:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882292:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441ae70. 00000400:00000010:1.0:1773941420.882294:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfaf00. 00000100:00000200:1.0:1773941420.882296:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882298:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a370. 00000400:00000010:1.0:1773941420.882300:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b84cfa6c0. 00000100:00000200:1.0:1773941420.882302:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882304:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a268. 00000400:00000010:1.0:1773941420.882307:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b990e6cc0. 00000100:00000200:1.0:1773941420.882309:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882311:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441abb0. 00000400:00000010:1.0:1773941420.882313:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b990e6000. 00000100:00000200:1.0:1773941420.882315:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882317:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441acb8. 00000400:00000010:1.0:1773941420.882319:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b990e6480. 00000100:00000200:1.0:1773941420.882321:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882322:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a738. 00000400:00000010:1.0:1773941420.882325:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b990e6300. 00000100:00000200:1.0:1773941420.882326:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882328:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a790. 00000400:00000010:1.0:1773941420.882330:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b990e6600. 00000100:00000200:1.0:1773941420.882332:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882339:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a108. 00000400:00000010:1.0:1773941420.882341:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b990e6780. 00000100:00000200:1.0:1773941420.882344:0:12135:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941420.882346:0:12135:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a630. 00000400:00000010:1.0:1773941420.882348:0:12135:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b990e66c0. 00000100:00000010:1.0:1773941420.882351:0:12135:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b8896a800. 00000100:00000200:1.0:1773941420.882370:0:12135:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000001:2.0:1773941420.882388:0:12130:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.882392:0:12130:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:2.0:1773941420.882393:0:12130:0:(service.c:3441:ptlrpc_start_thread()) ldlm_canceld[0] started 1 min 3 max 34 00000100:00000010:2.0:1773941420.882397:0:12130:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b89317000. 00000100:00100000:2.0:1773941420.882399:0:12130:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cn00_001' 00000100:00000001:2.0:1773941420.882544:0:12136:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:1.0:1773941420.882591:0:12136:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5b850b0b40. 00000020:00000010:1.0:1773941420.882595:0:12136:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b850b0bc0. 00000020:00000010:1.0:1773941420.882600:0:12136:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b839e5e00. 00000020:00000010:1.0:1773941420.882603:0:12136:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b8a0d1800. 00000020:00000010:1.0:1773941420.882606:0:12136:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5b880f9660. 00000040:00000010:1.0:1773941420.882608:0:12136:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b91d91400. 00000020:00000010:1.0:1773941420.882611:0:12136:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b8896b000. 00000020:00000010:1.0:1773941420.882614:0:12136:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b98a05f98. 00000020:00000010:1.0:1773941420.882617:0:12136:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b91d90400. 80000000:00000010:1.0:1773941420.882619:0:12136:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f98a0. 40000000:00000010:1.0:1773941420.882621:0:12136:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b850b0480. 00008000:00000010:1.0:1773941420.882631:0:12136:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d26350. 00100000:00000010:1.0:1773941420.882637:0:12136:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b8896d800. 00040000:00000010:1.0:1773941420.882640:0:12136:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b9024b700. 00040000:00000010:1.0:1773941420.882643:0:12136:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b8896f000. 00040000:00000010:1.0:1773941420.882645:0:12136:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b91d91800. 00000004:00000010:1.0:1773941420.882648:0:12136:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b8896f800. 00000004:00000010:1.0:1773941420.882652:0:12136:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b90f32000. 00000004:00000010:1.0:1773941420.882655:0:12136:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b8896c800. 00002000:00000010:1.0:1773941420.882657:0:12136:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0736400. 00000004:00000010:1.0:1773941420.882660:0:12136:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5ba0737800. 00000100:00000010:1.0:1773941420.882664:0:12136:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b8896a000. 00000100:00000200:1.0:1773941420.882676:0:12136:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000001:2.0:1773941420.882694:0:12130:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.882698:0:12130:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:2.0:1773941420.882699:0:12130:0:(service.c:3441:ptlrpc_start_thread()) ldlm_canceld[0] started 2 min 3 max 34 00000100:00000010:2.0:1773941420.882727:0:12130:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b89317400. 00000100:00100000:2.0:1773941420.882731:0:12130:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cn00_002' 00000400:00000200:2.0:1773941420.882811:0:2340:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000800:00000200:2.1:1773941420.882816:0:2340:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected e ksnr_max_conns 1 00000800:00000200:2.0:1773941420.882823:0:2340:0:(socklnd.c:1183:ksocknal_create_conn()) New conn 12345-192.168.204.153@tcp p 3.x 192.168.204.43 -> 192.168.204.153:988 incarnation:1773939896378632739 sched[1] 00000800:00000010:2.0:1773941420.882828:0:2340:0:(socklnd.c:1191:ksocknal_create_conn()) kfreed 'hello': 144 at ffff8b5ba0b540c0. 00000800:00000001:3.0:1773941420.882858:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000100:00000001:1.0:1773941420.882859:0:12137:0:(service.c:2998:ptlrpc_main()) Process entered 00000800:00000001:3.0:1773941420.882865:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1773941420.882871:0:12137:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5b850b0280. 00000020:00000010:1.0:1773941420.882874:0:12137:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b850b0e40. 00000020:00000010:1.0:1773941420.882877:0:12137:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b862a4e00. 00000020:00000010:1.0:1773941420.882883:0:12137:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b862a4400. 00000020:00000010:1.0:1773941420.882884:0:12137:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5b880f90c0. 00000040:00000010:1.0:1773941420.882886:0:12137:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b862a5400. 00000020:00000010:1.0:1773941420.882888:0:12137:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b8896e800. 00000020:00000010:1.0:1773941420.882890:0:12137:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b98a02d48. 00000020:00000010:1.0:1773941420.882892:0:12137:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b862a5000. 80000000:00000010:1.0:1773941420.882894:0:12137:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f9cc0. 40000000:00000010:1.0:1773941420.882895:0:12137:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b850b0200. 00008000:00000010:1.0:1773941420.882897:0:12137:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d27c08. 00100000:00000010:1.0:1773941420.882899:0:12137:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b8896d000. 00040000:00000010:1.0:1773941420.882901:0:12137:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b9024b500. 00040000:00000010:1.0:1773941420.882903:0:12137:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b8896e000. 00040000:00000010:1.0:1773941420.882904:0:12137:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b862a4800. 00000004:00000010:1.0:1773941420.882906:0:12137:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b88968000. 00000004:00000010:1.0:1773941420.882909:0:12137:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b90f31000. 00000004:00000010:1.0:1773941420.882911:0:12137:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b99ed1800. 00002000:00000010:1.0:1773941420.882913:0:12137:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5ba0735800. 00000004:00000010:1.0:1773941420.882915:0:12137:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5ba0736000. 00000100:00000010:1.0:1773941420.882919:0:12137:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b885e5800. 00000100:00000200:1.0:1773941420.882924:0:12137:0:(service.c:3086:ptlrpc_main()) service thread 2 (#3) started 00000100:00000001:3.0:1773941420.882945:0:12130:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.882946:0:12130:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:3.0:1773941420.882947:0:12130:0:(service.c:3441:ptlrpc_start_thread()) ldlm_canceld[1] started 0 min 3 max 34 00000100:00000010:3.0:1773941420.882952:0:12130:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b99105000. 00000100:00100000:3.0:1773941420.882954:0:12130:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cn01_000' 00000100:00000001:3.0:1773941420.883063:0:12138:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:3.0:1773941420.883075:0:12138:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5b85068cc0. 00000020:00000010:3.0:1773941420.883077:0:12138:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b850680c0. 00000020:00000010:3.0:1773941420.883080:0:12138:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b99104800. 00000020:00000010:3.0:1773941420.883082:0:12138:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba216bc00. 00000020:00000010:3.0:1773941420.883084:0:12138:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9f00. 00000040:00000010:3.0:1773941420.883086:0:12138:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba216a000. 00000020:00000010:3.0:1773941420.883092:0:12138:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b98ac2000. 00000020:00000010:3.0:1773941420.883094:0:12138:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b88498a10. 00000020:00000010:3.0:1773941420.883096:0:12138:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba216be00. 80000000:00000010:3.0:1773941420.883098:0:12138:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9300. 40000000:00000010:3.0:1773941420.883099:0:12138:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b85068100. 00008000:00000010:3.0:1773941420.883101:0:12138:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b98a1fc08. 00100000:00000010:3.0:1773941420.883103:0:12138:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b98ac4800. 00040000:00000010:3.0:1773941420.883105:0:12138:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98a69a00. 00040000:00000010:3.0:1773941420.883107:0:12138:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b98ac7800. 00040000:00000010:3.0:1773941420.883110:0:12138:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b84ed5200. 00000004:00000010:3.0:1773941420.883112:0:12138:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b98ac5000. 00000004:00000010:3.0:1773941420.883114:0:12138:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5ba0b8c000. 00000004:00000010:3.0:1773941420.883116:0:12138:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b98ac6800. 00002000:00000010:3.0:1773941420.883118:0:12138:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b90263800. 00000004:00000010:3.0:1773941420.883120:0:12138:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5b90260800. 00000100:00000200:3.0:1773941420.883122:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883125:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f873c8. 00000400:00000010:3.0:1773941420.883128:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90ec5000. 00000100:00000200:3.0:1773941420.883130:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883132:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f879a0. 00000400:00000010:3.0:1773941420.883133:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90ec5b40. 00000100:00000200:3.0:1773941420.883135:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883136:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f877e8. 00000400:00000010:3.0:1773941420.883137:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b90ec5480. 00000100:00000200:3.0:1773941420.883139:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883140:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87ec8. 00000400:00000010:3.0:1773941420.883142:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b91fc0cc0. 00000100:00000200:3.0:1773941420.883143:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883144:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87dc0. 00000400:00000010:3.0:1773941420.883145:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b91fc0900. 00000100:00000200:3.0:1773941420.883146:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883147:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87840. 00000400:00000010:3.0:1773941420.883149:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b91fc0e40. 00000100:00000200:3.0:1773941420.883150:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883154:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f878f0. 00000400:00000010:3.0:1773941420.883156:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b91fc0780. 00000100:00000200:3.0:1773941420.883157:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883158:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87000. 00000400:00000010:3.0:1773941420.883160:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b88ca6d80. 00000100:00000200:3.0:1773941420.883161:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883162:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f875d8. 00000400:00000010:3.0:1773941420.883164:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b88ca6cc0. 00000100:00000200:3.0:1773941420.883165:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883166:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87b58. 00000400:00000010:3.0:1773941420.883168:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed70c0. 00000100:00000200:3.0:1773941420.883169:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883170:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87f20. 00000400:00000010:3.0:1773941420.883172:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7480. 00000100:00000200:3.0:1773941420.883173:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883174:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87580. 00000400:00000010:3.0:1773941420.883175:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7540. 00000100:00000200:3.0:1773941420.883176:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883177:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87898. 00000400:00000010:3.0:1773941420.883178:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7780. 00000100:00000200:3.0:1773941420.883179:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883181:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87e70. 00000400:00000010:3.0:1773941420.883182:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7d80. 00000100:00000200:3.0:1773941420.883183:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883184:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87370. 00000400:00000010:3.0:1773941420.883185:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7180. 00000100:00000200:3.0:1773941420.883186:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883187:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87268. 00000400:00000010:3.0:1773941420.883188:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7c00. 00000100:00000200:3.0:1773941420.883189:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883191:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87bb0. 00000400:00000010:3.0:1773941420.883192:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7300. 00000100:00000200:3.0:1773941420.883193:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883194:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87cb8. 00000400:00000010:3.0:1773941420.883195:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7240. 00000100:00000200:3.0:1773941420.883199:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883201:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87738. 00000400:00000010:3.0:1773941420.883202:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7b40. 00000100:00000200:3.0:1773941420.883203:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883204:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87790. 00000400:00000010:3.0:1773941420.883205:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7840. 00000100:00000200:3.0:1773941420.883206:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883207:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87108. 00000400:00000010:3.0:1773941420.883208:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7000. 00000100:00000200:3.0:1773941420.883209:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883210:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87c60. 00000400:00000010:3.0:1773941420.883211:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed79c0. 00000100:00000200:3.0:1773941420.883212:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883213:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87210. 00000400:00000010:3.0:1773941420.883214:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7e40. 00000100:00000200:3.0:1773941420.883215:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883216:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87aa8. 00000400:00000010:3.0:1773941420.883217:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7f00. 00000100:00000200:3.0:1773941420.883218:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883219:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87058. 00000400:00000010:3.0:1773941420.883220:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7900. 00000100:00000200:3.0:1773941420.883221:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883222:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f876e0. 00000400:00000010:3.0:1773941420.883224:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed76c0. 00000100:00000200:3.0:1773941420.883225:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883226:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f87688. 00000400:00000010:3.0:1773941420.883227:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7cc0. 00000100:00000200:3.0:1773941420.883228:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883229:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f874d0. 00000400:00000010:3.0:1773941420.883230:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7600. 00000100:00000200:3.0:1773941420.883231:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883232:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f870b0. 00000400:00000010:3.0:1773941420.883233:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed73c0. 00000100:00000200:3.0:1773941420.883234:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883235:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b85f872c0. 00000400:00000010:3.0:1773941420.883239:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b85ed7a80. 00000100:00000200:3.0:1773941420.883241:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883242:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab840. 00000400:00000010:3.0:1773941420.883244:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b480. 00000100:00000200:3.0:1773941420.883245:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883246:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab8f0. 00000400:00000010:3.0:1773941420.883247:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680bb40. 00000100:00000200:3.0:1773941420.883248:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883249:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab000. 00000400:00000010:3.0:1773941420.883251:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b780. 00000100:00000200:3.0:1773941420.883252:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883253:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab5d8. 00000400:00000010:3.0:1773941420.883254:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b180. 00000100:00000200:3.0:1773941420.883255:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883256:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abb58. 00000400:00000010:3.0:1773941420.883257:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b0c0. 00000100:00000200:3.0:1773941420.883258:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883259:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abf20. 00000400:00000010:3.0:1773941420.883261:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680be40. 00000100:00000200:3.0:1773941420.883262:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883263:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab580. 00000400:00000010:3.0:1773941420.883273:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b3c0. 00000100:00000200:3.0:1773941420.883276:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883277:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab898. 00000400:00000010:3.0:1773941420.883279:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680bcc0. 00000100:00000200:3.0:1773941420.883280:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883281:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abe70. 00000400:00000010:3.0:1773941420.883282:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680bf00. 00000100:00000200:3.0:1773941420.883283:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883284:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab370. 00000400:00000010:3.0:1773941420.883286:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680bc00. 00000100:00000200:3.0:1773941420.883287:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883288:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab268. 00000400:00000010:3.0:1773941420.883289:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b9c0. 00000100:00000200:3.0:1773941420.883290:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883294:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abbb0. 00000400:00000010:3.0:1773941420.883296:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b900. 00000100:00000200:3.0:1773941420.883297:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883299:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abcb8. 00000400:00000010:3.0:1773941420.883300:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b240. 00000100:00000200:3.0:1773941420.883301:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883302:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab738. 00000400:00000010:3.0:1773941420.883304:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680ba80. 00000100:00000200:3.0:1773941420.883305:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883306:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab790. 00000400:00000010:3.0:1773941420.883307:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b8680b300. 00000100:00000200:3.0:1773941420.883308:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883309:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab108. 00000400:00000010:3.0:1773941420.883310:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b98124300. 00000100:00000200:3.0:1773941420.883312:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883313:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab630. 00000400:00000010:3.0:1773941420.883314:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b98124cc0. 00000100:00000200:3.0:1773941420.883315:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883316:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abb00. 00000400:00000010:3.0:1773941420.883317:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b98124780. 00000100:00000200:3.0:1773941420.883318:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883319:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab9f8. 00000400:00000010:3.0:1773941420.883320:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b98124a80. 00000100:00000200:3.0:1773941420.883321:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883323:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abd10. 00000400:00000010:3.0:1773941420.883324:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5b98124f00. 00000100:00000200:3.0:1773941420.883325:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883326:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abd68. 00000400:00000010:3.0:1773941420.883328:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e6c0. 00000100:00000200:3.0:1773941420.883329:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883330:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab948. 00000400:00000010:3.0:1773941420.883331:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5ec00. 00000100:00000200:3.0:1773941420.883332:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883333:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abc08. 00000400:00000010:3.0:1773941420.883334:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5ef00. 00000100:00000200:3.0:1773941420.883339:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883340:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845aba50. 00000400:00000010:3.0:1773941420.883341:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e780. 00000100:00000200:3.0:1773941420.883343:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883344:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab478. 00000400:00000010:3.0:1773941420.883345:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5ee40. 00000100:00000200:3.0:1773941420.883346:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883347:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab1b8. 00000400:00000010:3.0:1773941420.883348:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e3c0. 00000100:00000200:3.0:1773941420.883349:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883350:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab318. 00000400:00000010:3.0:1773941420.883351:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e000. 00000100:00000200:3.0:1773941420.883353:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883353:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab528. 00000400:00000010:3.0:1773941420.883355:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e180. 00000100:00000200:3.0:1773941420.883356:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883357:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab160. 00000400:00000010:3.0:1773941420.883358:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e540. 00000100:00000200:3.0:1773941420.883359:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883359:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abf78. 00000400:00000010:3.0:1773941420.883361:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e840. 00000100:00000200:3.0:1773941420.883362:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883363:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab420. 00000400:00000010:3.0:1773941420.883364:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5ea80. 00000100:00000200:3.0:1773941420.883365:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883366:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab2c0. 00000400:00000010:3.0:1773941420.883367:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5ecc0. 00000100:00000200:3.0:1773941420.883368:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883369:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab0b0. 00000400:00000010:3.0:1773941420.883370:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5eb40. 00000100:00000200:3.0:1773941420.883371:0:12138:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941420.883372:0:12138:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab688. 00000400:00000010:3.0:1773941420.883374:0:12138:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff8b5ba0b5e900. 00000100:00000010:3.0:1773941420.883377:0:12138:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b98ac1000. 00000100:00000200:3.0:1773941420.883391:0:12138:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000001:2.0:1773941420.883405:0:12130:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.883409:0:12130:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:2.0:1773941420.883410:0:12130:0:(service.c:3441:ptlrpc_start_thread()) ldlm_canceld[1] started 1 min 3 max 34 00000100:00000010:2.0:1773941420.883416:0:12130:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b91df4200. 00000100:00100000:2.0:1773941420.883419:0:12130:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cn01_001' 00000100:00000001:2.0:1773941420.883561:0:12139:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:2.0:1773941420.883581:0:12139:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5b845ad8c0. 00000020:00000010:2.0:1773941420.883584:0:12139:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b845ad400. 00000020:00000010:2.0:1773941420.883587:0:12139:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b91df4e00. 00000020:00000010:2.0:1773941420.883590:0:12139:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b91df4800. 00000020:00000010:2.0:1773941420.883593:0:12139:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0166ea0. 00000040:00000010:2.0:1773941420.883595:0:12139:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b91df5a00. 00000020:00000010:2.0:1773941420.883598:0:12139:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90f21800. 00000020:00000010:2.0:1773941420.883601:0:12139:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b8855e9a8. 00000020:00000010:2.0:1773941420.883604:0:12139:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b91df5e00. 80000000:00000010:2.0:1773941420.883606:0:12139:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0166720. 40000000:00000010:2.0:1773941420.883609:0:12139:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b845ad600. 00008000:00000010:2.0:1773941420.883612:0:12139:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b90d774f8. 00100000:00000010:2.0:1773941420.883615:0:12139:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b90f20000. 00040000:00000010:2.0:1773941420.883618:0:12139:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98bb0f00. 00040000:00000010:2.0:1773941420.883620:0:12139:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b90f23000. 00040000:00000010:2.0:1773941420.883621:0:12139:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b91df5200. 00000004:00000010:2.0:1773941420.883623:0:12139:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b90f22000. 00000004:00000010:2.0:1773941420.883626:0:12139:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5b88906000. 00000004:00000010:2.0:1773941420.883628:0:12139:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b90f27000. 00002000:00000010:2.0:1773941420.883630:0:12139:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b83f0b400. 00000004:00000010:2.0:1773941420.883632:0:12139:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5b83f0a800. 00000100:00000010:2.0:1773941420.883634:0:12139:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b90f20800. 00000100:00000200:2.0:1773941420.883642:0:12139:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000001:3.0:1773941420.883652:0:12130:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941420.883656:0:12130:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:3.0:1773941420.883657:0:12130:0:(service.c:3441:ptlrpc_start_thread()) ldlm_canceld[1] started 2 min 3 max 34 00000100:00000010:3.0:1773941420.883662:0:12130:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff8b5b84ed4a00. 00000100:00100000:3.0:1773941420.883665:0:12130:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cn01_002' 00000100:00000001:3.0:1773941420.883815:0:12140:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:3.0:1773941420.883828:0:12140:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff8b5b85068d80. 00000020:00000010:3.0:1773941420.883832:0:12140:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068c40. 00000020:00000010:3.0:1773941420.883835:0:12140:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84ed4e00. 00000020:00000010:3.0:1773941420.883840:0:12140:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b90c3da00. 00000020:00000010:3.0:1773941420.883842:0:12140:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9240. 00000040:00000010:3.0:1773941420.883845:0:12140:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b90c3ce00. 00000020:00000010:3.0:1773941420.883848:0:12140:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b98ac3800. 00000020:00000010:3.0:1773941420.883851:0:12140:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b8849f3b8. 00000020:00000010:3.0:1773941420.883854:0:12140:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b9a587600. 80000000:00000010:3.0:1773941420.883857:0:12140:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc97e0. 40000000:00000010:3.0:1773941420.883859:0:12140:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b850686c0. 00008000:00000010:3.0:1773941420.883863:0:12140:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8b5b98a1c710. 00100000:00000010:3.0:1773941420.883866:0:12140:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b98ac6000. 00040000:00000010:3.0:1773941420.883869:0:12140:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98a69100. 00040000:00000010:3.0:1773941420.883872:0:12140:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b98ac1800. 00040000:00000010:3.0:1773941420.883875:0:12140:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b98182400. 00000004:00000010:3.0:1773941420.883878:0:12140:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b98ac5800. 00000004:00000010:3.0:1773941420.883881:0:12140:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5ba0b89000. 00000004:00000010:3.0:1773941420.883885:0:12140:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5ba21ce800. 00002000:00000010:3.0:1773941420.883888:0:12140:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b90261400. 00000004:00000010:3.0:1773941420.883891:0:12140:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5b90261800. 00000100:00000010:3.0:1773941420.883895:0:12140:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff8b5b8907b000. 00000100:00000200:3.0:1773941420.883909:0:12140:0:(service.c:3086:ptlrpc_main()) service thread 2 (#3) started 00000100:00000001:2.0:1773941420.883917:0:12130:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.883920:0:12130:0:(service.c:3418:ptlrpc_start_threads()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.883922:0:12130:0:(service.c:857:ptlrpc_register_service()) Process leaving (rc=18446615823911764480 : -128249797787136 : ffff8b5b845c4200) 00010000:00000010:2.0:1773941420.883927:0:12130:0:(ldlm_lockd.c:3466:ldlm_setup()) kmalloced '(blp)': 168 at ffff8b5ba0b54300. 00010000:00000001:2.0:1773941420.884030:0:12141:0:(ldlm_lockd.c:2959:ldlm_bl_thread_main()) Process entered 00010000:00000010:2.0:1773941420.884033:0:12141:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at ffff8b5b845ad340. 00000020:00000010:2.0:1773941420.884035:0:12141:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b91df5800. 00000020:00000010:2.0:1773941420.884037:0:12141:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b91df4600. 00000020:00000010:2.0:1773941420.884039:0:12141:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90f26800. 00000020:00000010:2.0:1773941420.884041:0:12141:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b8855bc60. 00000020:00000010:2.0:1773941420.884043:0:12141:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b84d8a400. 80000000:00000010:2.0:1773941420.884050:0:12141:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0166360. 40000000:00000010:2.0:1773941420.884051:0:12141:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b845ad2c0. 00100000:00000010:2.0:1773941420.884053:0:12141:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b90f22800. 00040000:00000010:2.0:1773941420.884055:0:12141:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98bb0000. 00040000:00000010:2.0:1773941420.884057:0:12141:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b84d8b800. 00002000:00000010:2.0:1773941420.884059:0:12141:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b83f0b800. 00000020:00000010:2.0:1773941420.884061:0:12141:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b845ad480. 00000020:00000001:2.0:1773941420.884068:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941420.884069:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941420.884072:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1773941420.884073:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941420.884074:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941420.884075:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1773941420.884076:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941420.884077:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941420.884078:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1773941420.884200:0:12142:0:(ldlm_lockd.c:2959:ldlm_bl_thread_main()) Process entered 00010000:00000010:3.0:1773941420.884204:0:12142:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at ffff8b5b85068940. 00000020:00000010:3.0:1773941420.884209:0:12142:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b98183c00. 00000020:00000010:3.0:1773941420.884212:0:12142:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b98183400. 00000020:00000010:3.0:1773941420.884215:0:12142:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b8907e800. 00000020:00000010:3.0:1773941420.884223:0:12142:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b88498000. 00000020:00000010:3.0:1773941420.884227:0:12142:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b845d5c00. 80000000:00000010:3.0:1773941420.884230:0:12142:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9de0. 40000000:00000010:3.0:1773941420.884233:0:12142:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b85068700. 00100000:00000010:3.0:1773941420.884237:0:12142:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b90d97000. 00040000:00000010:3.0:1773941420.884240:0:12142:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98a69e00. 00040000:00000010:3.0:1773941420.884244:0:12142:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b91e55200. 00002000:00000010:3.0:1773941420.884248:0:12142:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b90260400. 00000020:00000010:3.0:1773941420.884251:0:12142:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068240. 00000020:00000001:3.0:1773941420.884261:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941420.884263:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941420.884278:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941420.884280:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941420.884282:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941420.884284:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941420.884286:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941420.884288:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941420.884290:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1773941420.884365:0:12143:0:(ldlm_lockd.c:159:expired_lock_main()) Process entered 00000020:00000010:3.0:1773941420.884369:0:12143:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b91e54c00. 00000020:00000010:3.0:1773941420.884373:0:12143:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b91ecb800. 00000040:00000010:3.0:1773941420.884376:0:12143:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b91eca600. 00000020:00000010:3.0:1773941420.884379:0:12143:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d96800. 00000020:00000010:3.0:1773941420.884382:0:12143:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff8b5b8849cb78. 00000020:00000010:3.0:1773941420.884385:0:12143:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b91ecbc00. 80000000:00000010:3.0:1773941420.884387:0:12143:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc92a0. 40000000:00000010:3.0:1773941420.884389:0:12143:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff8b5b85068800. 00100000:00000010:3.0:1773941420.884392:0:12143:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff8b5b90d92000. 00040000:00000010:3.0:1773941420.884395:0:12143:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98a69b00. 00040000:00000010:3.0:1773941420.884398:0:12143:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff8b5b90d96000. 00040000:00000010:3.0:1773941420.884400:0:12143:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b91ecae00. 00000004:00000010:3.0:1773941420.884403:0:12143:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff8b5b90d91800. 00000004:00000010:3.0:1773941420.884406:0:12143:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff8b5ba0b8e000. 00000004:00000010:3.0:1773941420.884409:0:12143:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff8b5b90d95000. 00002000:00000010:3.0:1773941420.884417:0:12143:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff8b5b90261c00. 00000004:00000010:3.0:1773941420.884420:0:12143:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff8b5b90263400. 00000020:00000010:3.0:1773941420.884422:0:12143:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068d40. 00010000:00000001:3.0:1773941420.884431:0:12143:0:(ldlm_lockd.c:142:have_expired_locks()) Process entered 00010000:00000001:3.0:1773941420.884433:0:12143:0:(ldlm_lockd.c:147:have_expired_locks()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941420.884437:0:12143:0:(ldlm_lockd.c:142:have_expired_locks()) Process entered 00010000:00000001:3.0:1773941420.884439:0:12143:0:(ldlm_lockd.c:147:have_expired_locks()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.884439:0:12130:0:(ldlm_pool.c:1348:ldlm_pools_init()) Process entered 00000001:00000010:2.0:1773941420.884444:0:12130:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at ffff8b5ba0166f00. 00000001:00000010:2.0:1773941420.884450:0:12130:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at ffff8b5ba01663c0. 00010000:00000001:2.0:1773941420.884454:0:12130:0:(ldlm_pool.c:1376:ldlm_pools_init()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.884464:0:12130:0:(ldlm_lockd.c:3518:ldlm_setup()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.884466:0:12130:0:(ldlm_lockd.c:3039:ldlm_get_ref()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941420.884470:0:12130:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at ffff8b5b90f25800. 00000020:00000040:2.0:1773941420.884472:0:12130:0:(obd_config.c:1030:class_incref()) incref MGC192.168.204.143@tcp (ffff8b5b982fb370) now 3 - import 00000020:00000001:2.0:1773941420.884476:0:12130:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941420.884477:0:12130:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=1 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941420.884479:0:12130:0:(genops.c:1249:class_import_put()) Process leaving 00010000:00000001:2.0:1773941420.884481:0:12130:0:(ldlm_lib.c:49:import_set_conn()) Process entered 00000100:00000200:2.0:1773941420.884489:0:12130:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.143@tcp->12345-192.168.204.143@tcp 00000100:00000001:2.0:1773941420.884492:0:12130:0:(connection.c:104:ptlrpc_connection_get()) Process entered 00000400:00000200:2.0:1773941420.884495:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941420.884500:0:12130:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.143@tcp: -114 00000400:00000200:2.0:1773941420.884501:0:12130:0:(peer.c:2727:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:2.0:1773941420.884503:0:12130:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:2.0:1773941420.884505:0:12130:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp: 0. pending discovery 00000400:00000200:2.0:1773941420.884507:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941420.884513:0:12130:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.143@tcp primary NID 192.168.204.143@tcp rc 0 00000100:00000001:2.0:1773941420.884516:0:12130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941420.884518:0:12130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 1 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941420.884519:0:12130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000100:00000001:2.0:1773941420.884521:0:12130:0:(connection.c:110:ptlrpc_connection_get()) Process leaving via out (rc=18446615824379267328 : -128249330284288 : 0xffff8b5ba039c900) 00000100:00000040:2.0:1773941420.884525:0:12130:0:(connection.c:149:ptlrpc_connection_get()) conn=ffff8b5ba039c900 refcount 1 to 192.168.204.143@tcp 00000100:00000040:2.0:1773941420.884527:0:12130:0:(client.c:137:ptlrpc_uuid_to_connection()) 192.168.204.143@tcp -> ffff8b5ba039c900 00010000:00000010:2.0:1773941420.884530:0:12130:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at ffff8b5ba0166540. 00010000:00080000:2.0:1773941420.884532:0:12130:0:(ldlm_lib.c:102:import_set_conn()) imp ffff8b5b90f25800@MGC192.168.204.143@tcp: add connection 192.168.204.143@tcp at head 00010000:00000001:2.0:1773941420.884535:0:12130:0:(ldlm_lib.c:109:import_set_conn()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.884538:0:12130:0:(ldlm_resource.c:938:ldlm_namespace_new()) Process entered 00010000:00000001:2.0:1773941420.884540:0:12130:0:(ldlm_lockd.c:3030:ldlm_get_ref()) Process entered 00010000:00000001:2.0:1773941420.884541:0:12130:0:(ldlm_lockd.c:3039:ldlm_get_ref()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.0:1773941420.884544:0:12130:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at ffff8b5b83f0b000. 00000001:00000001:2.0:1773941420.884547:0:12130:0:(hash.c:1038:cfs_hash_create()) Process entered 00000001:00000010:2.0:1773941420.884551:0:12130:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at ffff8b5b84d8b400. 00000001:00000010:2.0:1773941420.884557:0:12130:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 16 at ffff8b5b880e08c0. 00000001:00000010:2.0:1773941420.884561:0:12130:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 304 at ffff8b5b84d8b200. 00000001:00000010:2.0:1773941420.884563:0:12130:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 304 at ffff8b5b84d8a000. 00000020:00000010:2.0:1773941420.884601:0:12130:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5b83f0ac00. 00000020:00000010:2.0:1773941420.884603:0:12130:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at ffff8b5ba0166180. 00010000:00000001:2.0:1773941420.884624:0:12130:0:(ldlm_pool.c:890:ldlm_pool_init()) Process entered 00010000:00000001:2.0:1773941420.884639:0:12130:0:(ldlm_pool.c:809:ldlm_pool_debugfs_init()) Process entered 00000020:00000010:2.0:1773941420.884649:0:12130:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5b83f09400. 00000020:00000010:2.0:1773941420.884651:0:12130:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at ffff8b5b84d8b000. 00010000:00000001:2.0:1773941420.884659:0:12130:0:(ldlm_pool.c:864:ldlm_pool_debugfs_init()) Process leaving 00010000:00010000:2.0:1773941420.884661:0:12130:0:(ldlm_pool.c:924:ldlm_pool_init()) Lock pool ldlm-pool-MGC192.168.204.143@tcp-0 is initialized 00010000:00000001:2.0:1773941420.884662:0:12130:0:(ldlm_pool.c:926:ldlm_pool_init()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.884664:0:12130:0:(ldlm_resource.c:1046:ldlm_namespace_new()) Process leaving (rc=18446615823904714752 : -128249804836864 : ffff8b5b83f0b000) 00010000:00000001:2.0:1773941420.884665:0:12130:0:(ldlm_lib.c:605:client_obd_setup()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941420.884668:0:12130:0:(mgc_request.c:751:mgc_llog_init()) Process entered 00000040:00000001:2.0:1773941420.884669:0:12130:0:(llog_obd.c:133:llog_setup()) Process entered 00000040:00000010:2.0:1773941420.884671:0:12130:0:(llog_obd.c:26:llog_new_ctxt()) kmalloced '(ctxt)': 168 at ffff8b5ba0b54900. 00000020:00000040:2.0:1773941420.884676:0:12130:0:(genops.c:887:class_export_get()) GET export ffff8b5ba0b0c000 refcount=2 00000040:01000000:2.0:1773941420.884678:0:12130:0:(llog_obd.c:195:llog_setup()) obd MGC192.168.204.143@tcp ctxt 1 is initialized 00000040:00000001:2.0:1773941420.884679:0:12130:0:(llog_obd.c:199:llog_setup()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:2.0:1773941420.884681:0:12130:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000040:00000001:2.0:1773941420.884683:0:12130:0:(llog_net.c:31:llog_initiator_connect()) Process entered 00000020:00000040:2.0:1773941420.884685:0:12130:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=2 obd=MGC192.168.204.143@tcp 00000040:00000001:2.0:1773941420.884686:0:12130:0:(llog_net.c:44:llog_initiator_connect()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:2.0:1773941420.884687:0:12130:0:(lustre_log.h:400:llog_ctxt_put()) PUTting ctxt ffff8b5ba0b54900 : new refcount 1 10000000:00000001:2.0:1773941420.884689:0:12130:0:(mgc_request.c:767:mgc_llog_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941420.884791:0:12130:0:(lprocfs_status.c:1339:lprocfs_obd_setup()) Process leaving via already_registered (rc=0 : 0 : 0x0) 10000000:00000001:1.0:1773941420.884915:0:12144:0:(mgc_request.c:629:mgc_requeue_thread()) Process entered 10000000:01000000:1.0:1773941420.884918:0:12144:0:(mgc_request.c:631:mgc_requeue_thread()) Starting requeue thread 10000000:00000001:2.0:1773941420.884934:0:12130:0:(mgc_request.c:896:mgc_device_alloc()) Process leaving (rc=18446615824379266432 : -128249330285184 : ffff8b5ba039c580) 00000020:00000010:2.0:1773941420.884939:0:12130:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b84fee400. 00000040:00000010:2.0:1773941420.884943:0:12130:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b84feee00. 00000020:00000010:2.0:1773941420.884945:0:12130:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5ba0b0a800. 00000020:00000010:2.0:1773941420.884950:0:12130:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b84fee800. 00040000:00000010:2.0:1773941420.884953:0:12130:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b99101000. 00040000:00000010:2.0:1773941420.884955:0:12130:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b84fef200. 00000020:00000010:2.0:1773941420.884958:0:12130:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84fee000. 00000020:00000010:2.0:1773941420.884961:0:12130:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff8b5b8848f600. 00000020:00000010:2.0:1773941420.884965:0:12130:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff8b5b85eeb190. 00008000:00000010:2.0:1773941420.884967:0:12130:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8b5b902a3328. 00000020:00000010:2.0:1773941420.884970:0:12130:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84fef600. 00000020:00000001:2.0:1773941420.884972:0:12130:0:(obd_class.h:593:obd_setup()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941420.884973:0:12130:0:(obd_config.c:1030:class_incref()) incref MGC192.168.204.143@tcp (ffff8b5b982fb370) now 4 - setup 00000020:00000080:2.0:1773941420.884976:0:12130:0:(obd_config.c:861:class_setup()) finished setup of obd MGC192.168.204.143@tcp (uuid 1080e6b5-6a6c-4762-b0ec-c785990fec17) 00000020:00000001:2.0:1773941420.884977:0:12130:0:(obd_config.c:863:class_setup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941420.884979:0:12130:0:(obd_config.c:1651:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1773941420.884982:0:12130:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 104 at ffff8b5b8848fa00. 00000020:00000001:2.0:1773941420.884989:0:12130:0:(obd_class.h:528:obd_set_info_async()) Process entered 10000000:00000001:2.0:1773941420.884991:0:12130:0:(mgc_request.c:1055:mgc_set_info_async()) Process entered 10000000:00000001:2.0:1773941420.884994:0:12130:0:(mgc_request.c:1125:mgc_set_info_async()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941420.884995:0:12130:0:(obd_class.h:545:obd_set_info_async()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941420.884997:0:12130:0:(obd_class.h:852:obd_connect()) Process entered 00010000:00000001:2.0:1773941420.884998:0:12130:0:(ldlm_lib.c:670:client_connect_import()) Process entered 00000020:00000001:2.0:1773941420.885004:0:12130:0:(genops.c:1394:class_connect()) Process entered 00000020:00000001:2.0:1773941420.885006:0:12130:0:(genops.c:950:__class_new_export()) Process entered 00000020:00000010:2.0:1773941420.885009:0:12130:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff8b5ba0b0a800. 00000020:00000001:2.0:1773941420.885011:0:12130:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941420.885012:0:12130:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5ba0b0a800 with handle 0xa33a5be14eae1c44 to hash 00000020:00000001:2.0:1773941420.885014:0:12130:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00000020:00000001:2.0:1773941420.885017:0:12130:0:(obd_class.h:1008:obd_init_export()) Process entered 00000020:00000001:2.0:1773941420.885018:0:12130:0:(obd_class.h:1012:obd_init_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941420.885020:0:12130:0:(obd_config.c:1030:class_incref()) incref MGC192.168.204.143@tcp (ffff8b5b982fb370) now 5 - export 00000020:00000001:2.0:1773941420.885022:0:12130:0:(genops.c:1020:__class_new_export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000040:2.0:1773941420.885024:0:12130:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 1 00000020:00000080:2.0:1773941420.885025:0:12130:0:(genops.c:1404:class_connect()) connect: client 1080e6b5-6a6c-4762-b0ec-c785990fec17, cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941420.885027:0:12130:0:(genops.c:1405:class_connect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941420.885028:0:12130:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941420.885029:0:12130:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941420.885030:0:12130:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941420.885032:0:12130:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=2 00000020:00000001:2.0:1773941420.885033:0:12130:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941420.885034:0:12130:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000001:2.0:1773941420.885037:0:12130:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1773941420.885039:0:12130:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941420.885041:0:12130:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 0 00000400:00000200:2.0:1773941420.885045:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941420.885048:0:12130:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is not discovered: rc = -114 00000100:00080000:2.0:1773941420.885051:0:12130:0:(import.c:545:import_select_connection()) MGC192.168.204.143@tcp: skip NID 192.168.204.143@tcp as not ready: rc = -114 00000100:00000001:2.0:1773941420.885053:0:12130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941420.885054:0:12130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941420.885055:0:12130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:2.0:1773941420.885057:0:12130:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941420.885058:0:12130:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941420.885059:0:12130:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941420.885060:0:12130:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941420.885061:0:12130:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941420.885066:0:12130:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000001:2.0:1773941420.885068:0:12130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941420.885069:0:12130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941420.885071:0:12130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:2.0:1773941420.885072:0:12130:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:2.0:1773941420.885075:0:12130:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:2.0:1773941420.885076:0:12130:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941420.885078:0:12130:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941420.885080:0:12130:0:(sec.c:1432:sptlrpc_sec_create()) Process entered 02000000:08000000:2.0:1773941420.885081:0:12130:0:(sec.c:1450:sptlrpc_sec_create()) mgc MGC192.168.204.143@tcp: select security flavor null 02000000:00000001:2.0:1773941420.885083:0:12130:0:(sec.c:1471:sptlrpc_sec_create()) Process leaving (rc=18446744072653012640 : -1056538976 : ffffffffc1067ea0) 02000000:00000001:2.0:1773941420.885085:0:12130:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.885086:0:12130:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941420.885088:0:12130:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941420.885092:0:12130:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021d500. 00000020:00000040:2.0:1773941420.885093:0:12130:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 02000000:00000001:2.0:1773941420.885096:0:12130:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941420.885097:0:12130:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.885100:0:12130:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941420.885102:0:12130:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941420.885104:0:12130:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941420.885109:0:12130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b83f0bc00. 00000100:00000001:2.0:1773941420.885114:0:12130:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941420.885123:0:12130:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b9021d500 x1860112241144448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941420.885132:0:12130:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941420.885137:0:12130:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941420.885141:0:12130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1896fa] to pc [ptlrpcd_rcv+-1] req@ffff8b5b9021d500 x1860112241144448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941420.885158:0:12130:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.885162:0:12130:0:(pinger.c:365:ptlrpc_pinger_add_import()) Process entered 00000100:00080000:2.0:1773941420.885164:0:12130:0:(pinger.c:371:ptlrpc_pinger_add_import()) adding pingable import 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS 00000020:00000040:2.0:1773941420.885173:0:12130:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941420.885188:0:12130:0:(pinger.c:382:ptlrpc_pinger_add_import()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.885189:0:12130:0:(ldlm_lib.c:725:client_connect_import()) Process leaving 00000020:00000001:2.0:1773941420.885191:0:12130:0:(obd_class.h:868:obd_connect()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941420.885193:0:12130:0:(obd_mount.c:561:lustre_start_mgc()) kfreed 'uuid': 40 at ffff8b5ba039c7c0. 00000020:00000010:2.0:1773941420.885195:0:12130:0:(obd_mount.c:562:lustre_start_mgc()) kfreed 'data': 192 at ffff8b5ba0b6e000. 00000020:00000010:2.0:1773941420.885197:0:12130:0:(obd_mount.c:563:lustre_start_mgc()) kfreed 'mgcname': 23 at ffff8b5b83f0c460. 00000020:00000001:2.0:1773941420.885199:0:12130:0:(obd_mount.c:565:lustre_start_mgc()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1773941420.885201:0:12130:0:(llite_lib.c:1392:ll_fill_super()) Process entered 00000080:00200000:2.0:1773941420.885203:0:12130:0:(llite_lib.c:1395:ll_fill_super()) VFS Op: cfg_instance lustre-client-ffff8b5ba0b09800 (sb ffff8b5ba0b09800) 00000080:00000010:2.0:1773941420.885206:0:12130:0:(llite_lib.c:1399:ll_fill_super()) kmalloced '(cfg)': 88 at ffff8b5ba0166a20. 00000080:00000001:2.0:1773941420.885209:0:12130:0:(llite_lib.c:93:ll_init_sbi()) Process entered 00000080:00000010:2.0:1773941420.885213:0:12130:0:(llite_lib.c:95:ll_init_sbi()) kmalloced '(sbi)': 1376 at ffff8b5b84e42800. 00000100:00000040:1.0:1773941420.885214:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (196) 00000020:00000001:2.0:1773941420.885251:0:12130:0:(cl_page.c:1379:cl_cache_init()) Process entered 00000020:00000010:2.0:1773941420.885255:0:12130:0:(cl_page.c:1380:cl_cache_init()) kmalloced '(cache)': 152 at ffff8b5ba0b54e40. 00000020:00000001:2.0:1773941420.885257:0:12130:0:(cl_page.c:1396:cl_cache_init()) Process leaving (rc=18446615824387362368 : -128249322189248 : ffff8b5ba0b54e40) 00000080:00000010:2.0:1773941420.885261:0:12130:0:(llite_lib.c:129:ll_init_sbi()) kmalloced '(sbi->ll_foreign_symlink_prefix)': 6 at ffff8b5b84f33490. 00000080:00000010:2.0:1773941420.885277:0:12130:0:(llite_lib.c:136:ll_init_sbi()) kmalloced '(sbi->ll_foreign_symlink_upcall)': 5 at ffff8b5b84f33de0. 00000080:00000001:2.0:1773941420.885287:0:12130:0:(llite_lib.c:240:ll_init_sbi()) Process leaving (rc=18446615823920670720 : -128249788880896 : ffff8b5b84e42800) 00000080:00000001:2.0:1773941420.885290:0:12130:0:(llite_lib.c:1107:ll_options()) Process entered 00000080:00000001:2.0:1773941420.885291:0:12130:0:(llite_lib.c:1109:ll_options()) Process leaving (rc=0 : 0 : 0) 00000080:01000000:2.0:1773941420.885297:0:12130:0:(llite_lib.c:1429:ll_fill_super()) llite sb uuid: 57582858-f91e-4121-a177-bd923f96b6a2 00000080:00000001:2.0:1773941420.885435:0:12130:0:(lproc_llite.c:2772:ll_debugfs_register_super()) Process entered 00000400:00000200:0.0:1773941420.885660:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000010:0.0:1773941420.885668:0:2350:0:(peer.c:3405:lnet_peer_merge_data()) kfreed 'curnis': 40 at ffff8b5ba039cf00. 00000400:00000010:0.0:1773941420.885674:0:2350:0:(peer.c:3406:lnet_peer_merge_data()) kfreed 'addnis': 48 at ffff8b5ba039c640. 00000400:00000010:0.0:1773941420.885678:0:2350:0:(peer.c:3407:lnet_peer_merge_data()) kfreed 'delnis': 40 at ffff8b5ba039c9c0. 00000400:00000010:0.0:1773941420.885684:0:2350:0:(api-ni.c:1839:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at ffff8b5b902a1400. 00000400:00000200:0.0:1773941420.885692:0:2350:0:(peer.c:3410:lnet_peer_merge_data()) peer 192.168.204.143@tcp (ffff8b5b84fee600): 0 00000400:00000200:0.0:1773941420.885697:0:2350:0:(peer.c:3714:lnet_peer_data_present()) peer 192.168.204.143@tcp(ffff8b5b84fee600): 0. state = 0x100141 00000400:00000200:0.0:1773941420.885701:0:2350:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.143@tcp(ffff8b5b84fee600) state 0x100141 rc 1 00000400:00000200:0.0:1773941420.885768:0:2350:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.143@tcp(ffff8b5b84fee600) state 0x100141 00000400:00000010:0.0:1773941420.885792:0:2350:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6ce58. 00000400:00000010:0.0:1773941420.885808:0:2350:0:(lib-lnet.h:478:lnet_rspt_alloc()) alloc 'rspt': 56 at ffff8b5b8a087268. 00000400:00000200:0.0:1773941420.885814:0:2350:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.153@tcp 00000400:00000200:0.0:1773941420.885828:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:0.0:1773941420.885837:0:2350:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source Specified: 192.168.204.43@tcp to MR: 192.168.204.153@tcp local destination 00000400:00000200:0.0:1773941420.885846:0:2350:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [107:107:256:4294967295] 192.168.204.153@tcp LPNI seq info [2:2:8:4294967295] 00000400:00000200:0.0:1773941420.885852:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:0.0:1773941420.885860:0:2350:0:(lib-move.c:2009:lnet_handle_send()) rspt_next_hop_nid = 192.168.204.153@tcp 00000400:00000200:0.0:1773941420.885872:0:2350:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:192.168.204.43@tcp) -> 192.168.204.153@tcp(192.168.204.153@tcp:192.168.204.153@tcp) : PUT try# 0 00000020:00000010:2.0:1773941420.885878:0:12130:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5b83f08000. 00000020:00000010:2.0:1773941420.885884:0:12130:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 1440 at ffff8b5b84e43000. 00000800:00000200:0.0:1773941420.885891:0:2350:0:(socklnd_cb.c:1001:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:0.0:1773941420.885900:0:2350:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00000020:00000010:2.0:1773941420.885905:0:12130:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff8b5b83f08c00. 00000020:00000010:2.0:1773941420.885910:0:12130:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 576 at ffff8b5b83f0a400. 00000800:00000200:0.0:1773941420.885914:0:2350:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000080:00000001:2.0:1773941420.885923:0:12130:0:(lproc_llite.c:2842:ll_debugfs_register_super()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941420.885932:0:12130:0:(obd_mount.c:62:lustre_process_log()) Process entered 00000800:00000200:0.0:1773941420.885933:0:2350:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000020:00000010:2.0:1773941420.885936:0:12130:0:(obd_mount.c:67:lustre_process_log()) kmalloced '(bufs)': 104 at ffff8b5b89390780. 00000800:00000200:0.0:1773941420.885941:0:2350:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 144 niov 1 nkiov 1 00000020:00000010:2.0:1773941420.885942:0:12130:0:(obd_mount.c:76:lustre_process_log()) kmalloced '(lcfg)': 184 at ffff8b5ba0b54540. 00000020:00000001:2.0:1773941420.885949:0:12130:0:(obd_class.h:681:obd_process_config()) Process entered 00000020:00000010:2.0:1773941420.885954:0:12130:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84d8a600. 00000020:00000010:2.0:1773941420.885959:0:12130:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b84d8ac00. 00000400:00000200:0.0:1773941420.885967:0:2350:0:(peer.c:3922:lnet_peer_send_push()) peer 192.168.204.143@tcp 00000040:00000010:2.0:1773941420.885970:0:12130:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b84d8ba00. 00000400:00000200:0.0:1773941420.885970:0:2350:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.143@tcp(ffff8b5b84fee600) state 0x100541 rc 0 00000020:00000010:2.0:1773941420.885982:0:12130:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b84e46000. 00000020:00000010:2.0:1773941420.885988:0:12130:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b84d8aa00. 00040000:00000010:2.0:1773941420.885993:0:12130:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98bb0d00. 00040000:00000010:2.0:1773941420.885997:0:12130:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b84d8a800. 10000000:00000001:2.0:1773941420.885999:0:12130:0:(mgc_request.c:1932:mgc_process_config()) Process entered 10000000:00000001:2.0:1773941420.886001:0:12130:0:(mgc_request_server.c:668:mgc_process_config_server()) Process entered 10000000:01000000:2.0:1773941420.886003:0:12130:0:(mgc_request.c:1953:mgc_process_config()) parse_log lustre-client from 0 10000000:00000001:2.0:1773941420.886004:0:12130:0:(mgc_request.c:326:config_log_add()) Process entered 10000000:01000000:2.0:1773941420.886006:0:12130:0:(mgc_request.c:329:config_log_add()) add config log lustre-client-ffff8b5ba0b09800 10000000:00000001:2.0:1773941420.886009:0:12130:0:(mgc_request.c:159:config_log_find()) Process entered 10000000:00000001:2.0:1773941420.886010:0:12130:0:(mgc_request.c:177:config_log_find()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941420.886011:0:12130:0:(mgc_request.c:191:do_config_log_add()) Process entered 10000000:01000000:2.0:1773941420.886012:0:12130:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-sptlrpc-ffff8b5b982fb370 10000000:00000010:2.0:1773941420.886015:0:12130:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 287 at ffff8b5b84d8b600. 10000000:01000000:2.0:1773941420.886018:0:12130:0:(mgc_request.c:68:mgc_name2resid()) log lustre-sptlrpc to resid 0x65727473756c/0x0 (lustre) 00000020:00000040:2.0:1773941420.886020:0:12130:0:(genops.c:887:class_export_get()) GET export ffff8b5ba0b0c000 refcount=3 02000000:00000010:2.0:1773941420.886024:0:12130:0:(sec_config.c:546:sptlrpc_conf_get()) kmalloced '(conf)': 120 at ffff8b5b89390600. 02000000:08000000:2.0:1773941420.886026:0:12130:0:(sec_config.c:557:sptlrpc_conf_get()) create sptlrpc conf lustre 10000000:00000001:2.0:1773941420.886028:0:12130:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941420.886029:0:12130:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:2.0:1773941420.886031:0:12130:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941420.886032:0:12130:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941420.886036:0:12130:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021e300. 00000020:00000040:2.0:1773941420.886038:0:12130:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941420.886040:0:12130:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941420.886041:0:12130:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941420.886042:0:12130:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941420.886046:0:12130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b84d8ae00. 00000100:00000001:2.0:1773941420.886049:0:12130:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.886051:0:12130:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941420.886053:0:12130:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941420.886059:0:12130:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5ba0d64200. 00010000:00000001:2.0:1773941420.886062:0:12130:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941420.886065:0:12130:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5bc2150600. 00000020:00000010:2.0:1773941420.886073:0:12130:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff8b5b89390280. 00000020:00000001:2.0:1773941420.886075:0:12130:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941420.886076:0:12130:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5bc2150600 with handle 0xa33a5be14eae1c4b to hash 00000020:00000001:2.0:1773941420.886078:0:12130:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941420.886079:0:12130:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615824947283456 : -128248762268160 : ffff8b5bc2150600) 00010000:00000001:2.0:1773941420.886081:0:12130:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615824947283456 : -128248762268160 : ffff8b5bc2150600) 00010000:00000001:2.0:1773941420.886084:0:12130:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941420.886086:0:12130:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941420.886089:0:12130:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1c4b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941420.886096:0:12130:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1c4b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941420.886104:0:12130:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1c4b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941420.886107:0:12130:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941420.886109:0:12130:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941420.886113:0:12130:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b54240. 00000100:00000001:2.0:1773941420.886114:0:12130:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387359296 : -128249322192320 : ffff8b5ba0b54240) 00000100:00000001:2.0:1773941420.886117:0:12130:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941420.886118:0:12130:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109265664 : -128249600285952 : ffff8b5b9021e300) 00000100:00000001:2.0:1773941420.886122:0:12130:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941420.886124:0:12130:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.886126:0:12130:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941420.886128:0:12130:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941420.886135:0:12130:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021e300 x1860112241144576/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941420.886143:0:12130:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941420.886146:0:12130:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941420.886152:0:12130:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b5b9021e300 x1860112241144576/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941420.886172:0:12130:0:(client.c:1834:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.886175:0:12130:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941420.886177:0:12130:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941420.886179:0:12130:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b54240 going to sleep for 0 seconds 00000100:00000001:2.0:1773941420.886181:0:12130:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941420.886184:0:12130:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941420.886186:0:12130:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941420.886187:0:12130:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941420.886189:0:12130:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.886202:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941420.886204:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941420.886205:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941420.886208:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021d500 x1860112241144448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941420.886213:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941420.886214:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941420.886217:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941420.886219:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941420.886220:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941420.886223:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241144448:192.168.204.143@tcp:250:mount.lustre.0 00000100:00000001:2.0:1773941420.886225:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1773941420.886227:0:2351:0:(niobuf.c:917:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.886228:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.886230:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.886231:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941420.886234:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941420.886235:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941420.886237:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941420.886237:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941420.886239:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941420.886240:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941420.886253:0:761:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1773941420.886255:0:761:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941420.886258:0:761:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 448 at ffff8b5b84fef600. 00000020:00000001:2.0:1773941420.886288:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941420.886290:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941420.886292:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1773941420.886799:0:2346:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941420.886802:0:2346:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000400:00000200:2.0:1773941420.886807:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941420.886811:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941420.886816:0:2346:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:2.0:1773941420.886818:0:2346:0:(peer.c:3033:lnet_discovery_event_send()) Push Send to 192.168.204.153@tcp: 0 00000800:00000001:2.0:1773941420.887573:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941420.887583:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941420.887586:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941420.887593:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941420.887599:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941420.887604:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : ACK - for me 00000400:00000200:2.0:1773941420.887612:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941420.887615:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941420.887628:0:2347:0:(lib-move.c:4700:lnet_parse_ack()) 192.168.204.43@tcp: ACK from 12345-192.168.204.153@tcp into md 0x7e1 00000400:00000200:2.0:1773941420.887637:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941420.887644:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: ACK: OK 00000400:00000200:2.0:1773941420.887649:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6ce58 00000400:00000010:2.0:1773941420.887651:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6ce58. 00000400:00000200:2.0:1773941420.887655:0:2347:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 4 00000400:00000200:2.0:1773941420.887659:0:2347:0:(peer.c:2799:lnet_discovery_event_ack()) peer 192.168.204.143@tcp ev->status 0 00000400:00000200:2.0:1773941420.887664:0:2347:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.143@tcp: -114 00000800:00000001:2.0:1773941420.887683:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941420.887688:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941420.887689:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941420.887692:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941420.887696:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941420.887736:0:2347:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941420.887740:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941420.887742:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941420.887743:0:2350:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1773941420.887747:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941420.887757:0:2350:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.143@tcp(ffff8b5b84fee600) state 0x100141 00000400:00000200:2.0:1773941420.887760:0:2347:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 48 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1773941420.887764:0:2350:0:(peer.c:3844:lnet_peer_discovered()) peer 192.168.204.143@tcp 00000400:00000200:1.0:1773941420.887768:0:2350:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.143@tcp(ffff8b5b84fee600) state 0x100111 rc 0 00000400:00000200:2.0:1773941420.887773:0:2347:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 0 from 12345-192.168.204.143@tcp of length 48/48 into md 0x29 [1] + 0 00000400:00000200:1.0:1773941420.887776:0:2350:0:(peer.c:2421:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.204.143@tcp 00000400:00000010:2.0:1773941420.887778:0:2347:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8014b7e8. 00000400:00000200:2.0:1773941420.887784:0:2347:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md ffff8b5b85e6c220 00000800:00000001:2.0:1773941420.887788:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941420.887817:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941420.887819:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941420.887822:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941420.887826:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c220 00000400:00000010:2.0:1773941420.887828:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c220. 00000400:00000200:2.0:1773941420.887830:0:2347:0:(api-ni.c:2341:lnet_push_target_event_handler()) type 2 status 0 unlinked 1 00000400:00000200:2.0:1773941420.887834:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941420.887837:0:2347:0:(peer.c:2570:lnet_peer_push_event()) peer 192.168.204.143@tcp(ffff8b5b84fee600) is MR 00000400:00000010:2.0:1773941420.887840:0:2347:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at ffff8b5b845adf80. 00000400:00000200:2.0:1773941420.887845:0:2347:0:(peer.c:2644:lnet_peer_push_event()) Received Push 192.168.204.143@tcp 363512030 00000400:00000200:2.0:1773941420.887854:0:2347:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.143@tcp: 0 00000400:00000200:2.0:1773941420.887859:0:2347:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:2.0:1773941420.887863:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941420.887868:0:2347:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source Specified: 192.168.204.43@tcp to MR: 192.168.204.153@tcp local destination 00000400:00000200:2.0:1773941420.887881:0:2347:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [108:108:256:4294967295] 192.168.204.153@tcp LPNI seq info [3:3:8:4294967295] 00000400:00000200:2.0:1773941420.887887:0:2347:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941420.887887:0:2350:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1773941420.887894:0:2347:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:192.168.204.43@tcp) -> 192.168.204.153@tcp(192.168.204.153@tcp:192.168.204.153@tcp) 192.168.204.153@tcp : ACK try# 0 00000400:00000010:1.0:1773941420.887895:0:2350:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441ab00. 00000800:00000200:2.0:1773941420.887900:0:2347:0:(socklnd_cb.c:1001:ksocknal_send()) sending 0 bytes in 1 frags to 12345-192.168.204.153@tcp 00000400:00000010:1.0:1773941420.887902:0:2350:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1088. 00000800:00000010:2.0:1773941420.887904:0:2347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0e00. 00000400:00000200:1.0:1773941420.887910:0:2350:0:(api-ni.c:2331:lnet_push_target_post()) posted push target ffff8b5b84e8e000 00000800:00000200:2.0:1773941420.887916:0:2347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000400:00000200:1.0:1773941420.887922:0:2350:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.143@tcp(ffff8b5b84fee600) state 0x1000d1 00000800:00000200:2.0:1773941420.887927:0:2347:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000400:00000200:1.0:1773941420.887930:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000800:00000200:2.0:1773941420.887934:0:2347:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0e00 type 0, nob 96 niov 1 nkiov 0 00000400:00000200:1.0:1773941420.887943:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941420.887951:0:2350:0:(peer.c:1899:lnet_peer_add_nid()) peer 192.168.204.143@tcp NID 192.168.204.153@tcp flags 0x11: 0 00000400:00000010:1.0:1773941420.887956:0:2350:0:(peer.c:3265:lnet_peer_merge_data()) alloc '(curnis)': 40 at ffff8b5b850b0880. 00000400:00000010:1.0:1773941420.887961:0:2350:0:(peer.c:3266:lnet_peer_merge_data()) alloc '(addnis)': 48 at ffff8b5b850b0b80. 00000400:00000010:1.0:1773941420.887963:0:2350:0:(peer.c:3267:lnet_peer_merge_data()) alloc '(delnis)': 40 at ffff8b5b850b0fc0. 00000400:00000200:1.0:1773941420.887967:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941420.887972:0:2350:0:(peer.c:566:lnet_peer_del_nid()) peer 192.168.204.143@tcp NID 192.168.204.143@tcp flags 0x11: -1 00000400:00020000:1.0:1773941420.887974:0:2350:0:(peer.c:3365:lnet_peer_merge_data()) Error deleting NID 192.168.204.143@tcp from peer 192.168.204.143@tcp: -1 00000400:00000200:1.0:1773941420.887976:0:2350:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000010:1.0:1773941420.887978:0:2350:0:(peer.c:3405:lnet_peer_merge_data()) kfreed 'curnis': 40 at ffff8b5b850b0880. 00000400:00000010:1.0:1773941420.887982:0:2350:0:(peer.c:3406:lnet_peer_merge_data()) kfreed 'addnis': 48 at ffff8b5b850b0b80. 00000400:00000010:1.0:1773941420.887987:0:2350:0:(peer.c:3407:lnet_peer_merge_data()) kfreed 'delnis': 40 at ffff8b5b850b0fc0. 00000400:00000010:1.0:1773941420.887991:0:2350:0:(api-ni.c:1839:lnet_ping_buffer_free()) kfreed 'pbuf': 57 at ffff8b5b845adf80. 00000800:00000200:2.0:1773941420.887997:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000400:00000200:1.0:1773941420.888000:0:2350:0:(peer.c:3410:lnet_peer_merge_data()) peer 192.168.204.143@tcp (ffff8b5b84fee600): 0 00000800:00000010:2.0:1773941420.888003:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0e00. 00000400:00000200:1.0:1773941420.888004:0:2350:0:(peer.c:3714:lnet_peer_data_present()) peer 192.168.204.143@tcp(ffff8b5b84fee600): 0. state = 0x100151 00000400:00000200:1.0:1773941420.888011:0:2350:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.143@tcp(ffff8b5b84fee600) state 0x100151 rc 1 00000400:00000200:2.0:1773941420.888012:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941420.888020:0:2350:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.143@tcp(ffff8b5b84fee600) state 0x100151 00000400:00000200:2.0:1773941420.888023:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: ACK: OK 00000400:00000200:1.0:1773941420.888025:0:2350:0:(peer.c:3844:lnet_peer_discovered()) peer 192.168.204.143@tcp 00000400:00000200:1.0:1773941420.888026:0:2350:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.143@tcp(ffff8b5b84fee600) state 0x100111 rc 0 00000400:00000200:1.0:1773941420.888028:0:2350:0:(peer.c:2421:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.204.143@tcp 00000800:00000001:2.0:1773941420.888037:0:2347:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941420.888041:0:2347:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941421.919887:0:12130:0:(client.c:2542:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1773941421.919888:0:2351:0:(client.c:2542:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1773941421.919895:0:2351:0:(client.c:2460:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1773941421.919896:0:12130:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941421.919899:0:12130:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941421.919905:0:12130:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b54240 going to sleep for 0 seconds 00000100:00000001:1.0:1773941421.919910:0:12130:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941421.919914:0:12130:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941421.919915:0:2351:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1773941420/real 0] req@ffff8b5b9021d500 x1860112241144448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1773941421 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941421.919917:0:761:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1773941421.919918:0:12130:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941421.919923:0:12130:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941421.919928:0:12130:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941421.919934:0:761:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.919935:0:2351:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.919946:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941421.919950:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941421.919957:0:2351:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b5b9021d500 x1860112241144448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1773941421 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941421.919966:0:2351:0:(client.c:2514:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.919975:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941421.919977:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.919983:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1773941421.919985:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000100:00000001:0.0:1773941421.919986:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1773941421.919989:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941421.919992:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1773941421.920019:0:2351:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941421.920024:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021d500 x1860112241144448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1773941421 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941421.920030:0:2351:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1773941421.920032:0:2351:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.920033:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941421.920034:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.920036:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1773941421.920038:0:2351:0:(import.c:1084:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1773941421.920041:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1773941421.920058:0:2351:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.143@tcp failed (-110) 00000100:00000001:0.0:1773941421.920060:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1773941421.920063:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021d500 x1860112241144448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1773941421 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941421.920069:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241144448:192.168.204.143@tcp:250:mount.lustre.0 00000100:00000001:0.0:1773941421.920072:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.920074:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941421.920077:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021d500 x1860112241144448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1773941421 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941421.920082:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1773941421.920085:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941421.920088:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941421.920090:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941421.920094:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b83f0bc00. 00000100:00080000:1.0:1773941421.920096:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 0 suppress 0 02000000:00000001:0.0:1773941421.920099:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941421.920102:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:1.0:1773941421.920103:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000010:0.0:1773941421.920104:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021d500. 00000100:00000001:1.0:1773941421.920106:0:37:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1773941421.920109:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:1.0:1773941421.920110:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941421.920111:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.920113:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.920116:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941421.920122:0:37:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1773941421.920123:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.920127:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941421.920129:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941421.920130:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 191 00000400:00000200:1.0:1773941421.920139:0:37:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941421.920146:0:37:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941421.920151:0:37:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941421.920154:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941421.920156:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941421.920159:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941421.920162:0:37:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941421.920165:0:37:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941421.920168:0:37:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941421.920171:0:37:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941421.920173:0:37:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941421.920176:0:37:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941421.920179:0:37:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941421.920180:0:37:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941421.920182:0:37:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941421.920184:0:37:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941421.920188:0:37:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941421.920191:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941421.920194:0:37:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941421.920197:0:37:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941421.920200:0:37:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941421.920204:0:37:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941421.920207:0:37:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941421.920210:0:37:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941421.920224:0:37:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60000. 00000020:00000040:1.0:1773941421.920228:0:37:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941421.920232:0:37:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941421.920233:0:37:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941421.920236:0:37:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941421.920238:0:37:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941421.920257:0:37:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941421.920266:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0736800. 00000100:00000001:1.0:1773941421.920273:0:37:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941421.920280:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60000 x1860112241144704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941421.920288:0:37:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941421.920292:0:37:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941421.920296:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000b65cec4] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60000 x1860112241144704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941421.920309:0:37:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941421.920314:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (197) 00000100:00000001:0.0:1773941421.920331:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941421.920333:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941421.920334:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941421.920338:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60000 x1860112241144704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941421.920343:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941421.920344:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941421.920347:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941421.920350:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941421.920351:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941421.920354:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241144704:192.168.204.143@tcp:250:kworker.0 00000100:00000001:0.0:1773941421.920356:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941421.920360:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941421.920362:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941421.920364:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941421.920368:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0000. 02000000:00000001:0.0:1773941421.920369:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941421.920374:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941421.920377:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941421.920382:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941421.920389:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241144704, portal 25 00000100:00000001:0.0:1773941421.920391:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941421.920392:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121069568 : -128249588482048 : ffff8b5b90d60000) 00000100:00000040:0.0:1773941421.920396:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60000 x1860112241144704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941421.920400:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941421.920403:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941421.920405:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6caa0. 00000100:00000200:0.0:1773941421.920408:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241144704, offset 0 00000400:00000200:0.0:1773941421.920413:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941421.920420:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941421.920430:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941421.920435:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941421.920439:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941421.920443:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941421.920445:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941421.920447:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[0, 3] 00000400:00000200:0.0:1773941421.920450:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941421.920453:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [109:109:256:4294967295] 192.168.204.143@tcp LPNI seq info [4:4:8:4294967295] 00000400:00000200:0.0:1773941421.920456:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941421.920465:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941421.920470:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941421.920474:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00000800:00000200:0.0:1773941421.920481:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941421.920489:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941421.920493:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:0.0:1773941421.920506:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941421.920510:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941421.920512:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294862283 00000100:00000001:0.0:1773941421.920518:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294861166 00000100:00000001:0.0:1773941421.920520:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294862283 00000100:00000001:0.0:1773941421.920523:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294861166 00000100:00000001:0.0:1773941421.920525:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941421.920527:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.920529:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.920531:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.920534:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941421.920535:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1773941421.920537:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941421.920539:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941421.920541:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.920542:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941421.920639:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941421.920644:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000400:00000200:2.0:1773941421.920651:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941421.920660:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941421.920666:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6caa0 00000400:00000010:2.0:1773941421.920669:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6caa0. 00000100:00000001:2.0:1773941421.920674:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941421.920682:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60000 x1860112241144704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941421.920693:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941421.920696:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60000 x1860112241144704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941431 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941421.920728:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941421.920731:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941421.922258:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941421.922274:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941421.922277:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941421.922281:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941421.922293:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941421.922300:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941421.922308:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941421.922311:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941421.922317:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941421.922321:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321603380 00000400:00000200:2.0:1773941421.922325:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941421.922331:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xabd [1] + 0 00000800:00000001:2.0:1773941421.922339:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941421.922348:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941421.922350:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941421.922359:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941421.922377:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941421.922382:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60000 x1860112241144704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941431 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941421.922392:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60000 x1860112241144704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941431 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941421.922412:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941421.922419:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941421.922423:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.922452:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941421.922456:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941421.922462:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941421.922463:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941421.922466:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941421.922470:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941421.922471:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941421.922474:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941421.922479:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60000 x1860112241144704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941431 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941421.922487:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60000 x1860112241144704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941431 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941421.922492:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941421.922493:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.922495:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941421.922498:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941421.922500:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941421.922502:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941421.922504:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941421.922511:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=1773941421 v=5 (0 0 0 0) 00000100:00001000:0.0:1773941421.922516:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941421 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941421.922519:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000100:00000001:0.0:1773941421.922522:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941421.922523:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941421.922527:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941421.922529:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.922533:0:2351:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00100000:0.0:1773941421.922535:0:2351:0:(client.c:3081:ptlrpc_free_committed()) MGC192.168.204.143@tcp: committing for last_committed 0 gen 1 00000100:00000040:0.0:1773941421.922538:0:2351:0:(client.c:3191:ptlrpc_free_committed()) MGC192.168.204.143@tcp: normal: skip=0 replay=0/0 committed=0/0 00000100:00000001:0.0:1773941421.922540:0:2351:0:(client.c:3192:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1773941421.922542:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941421.922546:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60000 x1860112241144704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941431 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941421.922553:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941421.922562:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941421.922565:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.922567:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941421.922573:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941421.922576:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941421.922578:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941421.922580:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941421.922583:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941421.922585:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941421.922587:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941421.922591:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941421 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941421.922594:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000100:00080000:0.0:1773941421.922600:0:2351:0:(import.c:951:ptlrpc_connect_set_flags()) MGC192.168.204.143@tcp: Resetting ns_connect_flags to server flags: 0xa000011001002060 00000020:00000040:0.0:1773941421.922604:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941421.922606:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941421.922610:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941421.922613:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.922615:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941421.922618:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to FULL 00000100:00000001:0.0:1773941421.922621:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941421.922622:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941421.922624:0:2351:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941421.922640:0:2351:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941421.922642:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.922643:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941421.922645:0:2351:0:(import.c:1232:ptlrpc_connect_interpret()) Process leaving via finish (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941421.922648:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941421.922654:0:2351:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 000000006bbe330b): req@ffff8b5b9021e300 x1860112241144576/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941421.922673:0:2351:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.922677:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941421.922681:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60000 x1860112241144704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941431 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941421.922701:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241144704:192.168.204.143@tcp:250:kworker.0 00000100:00000001:2.0:1773941421.922744:0:12130:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941421.922750:0:12130:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941421.922754:0:12130:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941421.922760:0:12130:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941421.922766:0:12130:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1773941421.922767:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941421.922771:0:12130:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941421.922773:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941421.922778:0:12130:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941421.922781:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60000 x1860112241144704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941431 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941421.922783:0:12130:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941421.922792:0:12130:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1773941421.922794:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941421.922796:0:12130:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941421.922799:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1773941421.922801:0:12130:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941421.922805:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0000. 02000000:00000010:2.0:1773941421.922810:0:12130:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b83f08400. 02000000:00000001:0.0:1773941421.922812:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:2.0:1773941421.922816:0:12130:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941421.922817:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941421.922820:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941421.922825:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000200:2.0:1773941421.922826:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 02000000:00000010:0.0:1773941421.922829:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0736800. 00000400:00000010:2.0:1773941421.922833:0:12130:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9dc0. 02000000:00000001:0.0:1773941421.922836:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941421.922839:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000400:00000010:2.0:1773941421.922840:0:12130:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34ee0. 00000100:00000010:0.0:1773941421.922843:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60000. 00000100:00000001:0.0:1773941421.922849:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000200:2.0:1773941421.922850:0:12130:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241144576, portal 25 00000100:00000001:0.0:1773941421.922852:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941421.922856:0:12130:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941421.922858:0:12130:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109265664 : -128249600285952 : ffff8b5b9021e300) 00000100:00000001:0.0:1773941421.922866:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941421.922868:0:12130:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021e300 x1860112241144576/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941437 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941421.922877:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941421.922880:0:12130:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1773941421.922881:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.922888:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1773941421.922889:0:12130:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000100:00000001:0.0:1773941421.922894:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1773941421.922896:0:12130:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34a18. 00000100:00000200:2.0:1773941421.922901:0:12130:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241144576, offset 0 00000400:00000200:2.0:1773941421.922907:0:12130:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941421.922920:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941421.922925:0:12130:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941421.922930:0:12130:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941421.922935:0:12130:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941421.922940:0:12130:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941421.922944:0:12130:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941421.922948:0:12130:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[4, 3] 00000400:00000200:2.0:1773941421.922953:0:12130:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:2.0:1773941421.922960:0:12130:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [110:110:256:4294967295] 192.168.204.153@tcp LPNI seq info [5:5:8:4294967295] 00000400:00000200:2.0:1773941421.922966:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941421.922980:0:12130:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:2.0:1773941421.922985:0:12130:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:2.0:1773941421.922990:0:12130:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0c00. 00000800:00000200:2.0:1773941421.922996:0:12130:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:2.0:1773941421.923002:0:12130:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:2.0:1773941421.923006:0:12130:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0c00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941421.923022:0:12130:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941421.923026:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941421.923029:0:12130:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294862286 00000100:00000001:2.0:1773941421.923037:0:12130:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294862283 00000100:00000001:2.0:1773941421.923043:0:12130:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294862286 00000100:00000001:2.0:1773941421.923050:0:12130:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294862283 00000100:00000001:2.0:1773941421.923053:0:12130:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941421.923056:0:12130:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941421.923059:0:12130:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941421.923061:0:12130:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941421.923064:0:12130:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b54240 going to sleep for 11 seconds 00000100:00000001:2.0:1773941421.923068:0:12130:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941421.923071:0:12130:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941421.923095:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941421.923101:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0c00. 00000400:00000200:3.0:1773941421.923105:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941421.923111:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941421.923116:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34a18 00000400:00000010:3.0:1773941421.923117:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34a18. 00000100:00000001:3.0:1773941421.923121:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941421.923125:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021e300 x1860112241144576/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941437 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941421.923131:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941421.923133:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b9021e300 x1860112241144576/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941437 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941421.923138:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941421.923170:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941421.924096:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941421.924103:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941421.924109:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941421.924111:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941421.924114:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941421.924117:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941421.924121:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941421.924123:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941421.924125:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941421.924127:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321603300 00000400:00000200:2.0:1773941421.924129:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941421.924132:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xac5 [1] + 224 00000800:00000001:2.0:1773941421.924135:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941421.924139:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941421.924141:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941421.924143:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941421.924146:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941421.924149:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021e300 x1860112241144576/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941437 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941421.924156:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b9021e300 x1860112241144576/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941437 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941421.924170:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941421.924174:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941421.924175:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941421.924207:0:12130:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941421.924213:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941421.924214:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941421.924217:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9dc0. 00000400:00000200:1.0:1773941421.924221:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34ee0 00000400:00000010:1.0:1773941421.924223:0:12130:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34ee0. 00000100:00000001:1.0:1773941421.924226:0:12130:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941421.924231:0:12130:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021e300 x1860112241144576/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941437 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941421.924242:0:12130:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021e300 x1860112241144576/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941437 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941421.924261:0:12130:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941421.924263:0:12130:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941421.924266:0:12130:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941421.924276:0:12130:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941421.924278:0:12130:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941421.924280:0:12130:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941421.924283:0:12130:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941421.924289:0:12130:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=0 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941421.924292:0:12130:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941421.924295:0:12130:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941421.924299:0:12130:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941421.927018:0:12130:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941421.927035:0:12130:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941421.927040:0:12130:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941421.927045:0:12130:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941421.927050:0:12130:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941421.930343:0:12130:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941421.930347:0:12130:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941421.930348:0:12130:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941421.930349:0:12130:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941421.930356:0:12130:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b9021e300 x1860112241144576/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941437 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941421.930364:0:12130:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941421.930365:0:12130:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941421.930367:0:12130:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941421.930369:0:12130:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941421.930371:0:12130:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941421.930373:0:12130:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941421.930374:0:12130:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941421.930375:0:12130:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941421.930376:0:12130:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941421.930377:0:12130:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941421.930378:0:12130:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941421.930380:0:12130:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941421.930381:0:12130:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941421.930383:0:12130:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941421.930385:0:12130:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 192 00000400:00000200:1.0:1773941421.930391:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941421.930395:0:12130:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941421.930398:0:12130:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941421.930400:0:12130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941421.930401:0:12130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941421.930403:0:12130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941421.930405:0:12130:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941421.930419:0:12130:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941421.930421:0:12130:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941421.930423:0:12130:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941421.930426:0:12130:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941421.930428:0:12130:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941421.930430:0:12130:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941421.930431:0:12130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941421.930432:0:12130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941421.930433:0:12130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941421.930435:0:12130:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941421.930437:0:12130:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941421.930439:0:12130:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941421.930441:0:12130:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941421.930444:0:12130:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941421.930446:0:12130:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941421.930447:0:12130:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941421.930448:0:12130:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941421.930452:0:12130:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60380. 00000020:00000040:1.0:1773941421.930454:0:12130:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941421.930457:0:12130:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941421.930458:0:12130:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941421.930461:0:12130:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941421.930463:0:12130:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941421.930466:0:12130:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941421.930472:0:12130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737000. 00000100:00000001:1.0:1773941421.930476:0:12130:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941421.930481:0:12130:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60380 x1860112241144832/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941421.930488:0:12130:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941421.930491:0:12130:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941421.930494:0:12130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077e2ee70] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60380 x1860112241144832/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941421.930516:0:12130:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941421.930519:0:12130:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941421.930521:0:12130:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941421.930524:0:12130:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941421.930527:0:12130:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941421.930529:0:12130:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941421.930532:0:12130:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b54240 going to sleep for 1 seconds 00000100:00000001:1.0:1773941421.930534:0:12130:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941421.930537:0:12130:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941421.930543:0:12130:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021e300 x1860112241144576/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941437 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941421.930551:0:12130:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941421.930554:0:12130:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941421.930556:0:12130:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941421.930559:0:12130:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941421.930562:0:12130:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021e300 x1860112241144576/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941437 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941421.930571:0:12130:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021e300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:1080e6b5-6a6c-4762-b0ec-c785990fec17:12130:1860112241144576:192.168.204.143@tcp:101:mount.lustre.0 00000100:00000001:1.0:1773941421.930575:0:12130:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.930575:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941421.930585:0:12130:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941421.930586:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941421.930593:0:12130:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941421.930593:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941421.930606:0:12130:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941421.930613:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60380 x1860112241144832/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941421.930615:0:12130:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021e300 x1860112241144576/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941437 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941421.930648:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941421.930652:0:12130:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.930653:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941421.930659:0:12130:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b54240. 02000000:00000001:0.0:1773941421.930662:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941421.930664:0:12130:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941421.930667:0:12130:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:0.0:1773941421.930668:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941421.930670:0:12130:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:0.0:1773941421.930674:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941421.930679:0:12130:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941421.930680:0:12130:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941421.930682:0:12130:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5bc2150600 refcount=4 00000020:00000001:1.0:1773941421.930683:0:12130:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824947283456 : -128248762268160 : ffff8b5bc2150600) 00010000:00000001:1.0:1773941421.930686:0:12130:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615824947283456 : -128248762268160 : ffff8b5bc2150600) 00010000:00010000:1.0:1773941421.930690:0:12130:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1c4b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1773941421.930698:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241144832:192.168.204.143@tcp:250:mount.lustre.0 00010000:00000001:1.0:1773941421.930700:0:12130:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941421.930740:0:12130:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1c4b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941421.930750:0:12130:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:0.0:1773941421.930752:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00010000:1.0:1773941421.930758:0:12130:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1c4b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941421.930759:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941421.930763:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941421.930769:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941421.930777:0:12130:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000010:0.0:1773941421.930777:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0000. 00010000:00000001:1.0:1773941421.930782:0:12130:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 02000000:00000001:0.0:1773941421.930783:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941421.930791:0:12130:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1c4b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941421.930797:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941421.930805:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00000001:1.0:1773941421.930814:0:12130:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941421.930817:0:12130:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1773941421.930818:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00010000:00000001:1.0:1773941421.930820:0:12130:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941421.930823:0:12130:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1c4b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:0.0:1773941421.930826:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241144832, portal 25 00010000:00010000:1.0:1773941421.930831:0:12130:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5bc2150600 already unused, calling callback (ffffffffc1482c20) 00000100:00000001:0.0:1773941421.930832:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 10000000:00000001:1.0:1773941421.930840:0:12130:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:0.0:1773941421.930841:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121070464 : -128249588481152 : ffff8b5b90d60380) 10000000:00010000:1.0:1773941421.930847:0:12130:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1c4b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941421.930855:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60380 x1860112241144832/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941421.930872:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941421.930879:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941421.930880:0:12130:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941421.930883:0:12130:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000010:0.0:1773941421.930884:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c3b8. 00000020:00000001:1.0:1773941421.930885:0:12130:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941421.930889:0:12130:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5bc2150600 refcount=5 00000100:00000200:0.0:1773941421.930889:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241144832, offset 0 00000020:00000001:1.0:1773941421.930895:0:12130:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824947283456 : -128248762268160 : ffff8b5bc2150600) 00010000:00000001:1.0:1773941421.930898:0:12130:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615824947283456 : -128248762268160 : ffff8b5bc2150600) 00010000:00000001:1.0:1773941421.930903:0:12130:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:0.0:1773941421.930904:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941421.930906:0:12130:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1c4b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941421.930916:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:1.0:1773941421.930917:0:12130:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941421.930927:0:12130:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1c4b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941421.930935:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941421.930955:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941421.930964:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 10000000:01000000:1.0:1773941421.930965:0:12130:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:0.0:1773941421.930972:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000040:1.0:1773941421.930980:0:12130:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:0.0:1773941421.930981:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000001:1.0:1773941421.930990:0:12130:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941421.930992:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[4, 5] 00000400:00000200:0.0:1773941421.931002:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00010000:1.0:1773941421.931003:0:12130:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941421.931007:0:12130:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:0.0:1773941421.931007:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [111:111:256:4294967295] 192.168.204.143@tcp LPNI seq info [6:6:8:4294967295] 00000400:00000200:0.0:1773941421.931015:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941421.931016:0:12130:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941421.931020:0:12130:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941421.931023:0:12130:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941421.931029:0:12130:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941421.931034:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000020:00000040:1.0:1773941421.931039:0:12130:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5bc2150600 with handle 0xa33a5be14eae1c4b from hash 00000800:00000200:0.0:1773941421.931045:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941421.931047:0:12130:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941421.931050:0:12130:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941421.931054:0:12130:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941421.931056:0:12130:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000800:00000010:0.0:1773941421.931057:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00010000:00000001:1.0:1773941421.931060:0:12130:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941421.931064:0:12130:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000200:0.0:1773941421.931068:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941421.931071:0:12130:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941421.931075:0:12130:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941421.931079:0:12130:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941421.931081:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941421.931085:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:1.0:1773941421.931097:0:12130:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941421.931101:0:12130:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1c4b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941421.931106:0:12130:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941421.931107:0:12130:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941421.931109:0:12130:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941421.931110:0:12130:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:0.0:1773941421.931111:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941421.931115:0:12130:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941421.931120:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941421.931121:0:12130:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941421.931124:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294862294 00010000:00000001:1.0:1773941421.931125:0:12130:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941421.931130:0:12130:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1c4b lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941421.931134:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294862286 00000100:00000001:0.0:1773941421.931143:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294862294 00000100:00000001:0.0:1773941421.931147:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294862286 00000020:00000010:1.0:1773941421.931148:0:12130:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff8b5b90edb500. 00000100:00000001:0.0:1773941421.931150:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941421.931154:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1773941421.931155:0:12130:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5ba0d64200 count: 0 00000100:00000001:0.0:1773941421.931157:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.931165:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941421.931174:0:12130:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941421.931175:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941421.931179:0:12130:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941421.931180:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1773941421.931187:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941421.931189:0:12130:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941421.931191:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941421.931195:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941421.931199:0:12130:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021e300 x1860112241144576/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941437 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941421.931200:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941421.931214:0:12130:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941421.931217:0:12130:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941421.931220:0:12130:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b83f08400. 00000800:00000200:2.0:1773941421.931222:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 02000000:00000001:1.0:1773941421.931225:0:12130:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000800:00000010:2.0:1773941421.931228:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000020:00000001:1.0:1773941421.931229:0:12130:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941421.931233:0:12130:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000400:00000200:2.0:1773941421.931236:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:1.0:1773941421.931237:0:12130:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941421.931241:0:12130:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b84d8ae00. 02000000:00000001:1.0:1773941421.931257:0:12130:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:2.0:1773941421.931258:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 02000000:00000001:1.0:1773941421.931261:0:12130:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941421.931264:0:12130:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021e300. 00000400:00000200:2.0:1773941421.931266:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c3b8 00000100:00000001:1.0:1773941421.931267:0:12130:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941421.931269:0:12130:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941421.931272:0:12130:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000010:2.0:1773941421.931273:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c3b8. 10000000:01000000:1.0:1773941421.931274:0:12130:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:2.0:1773941421.931280:0:2347:0:(events.c:34:request_out_callback()) Process entered 10000000:00000001:1.0:1773941421.931281:0:12130:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941421.931285:0:12130:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0fc0. 00000100:00000200:2.0:1773941421.931288:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60380 x1860112241144832/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941421.931289:0:12130:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b6400. 00000020:00000010:1.0:1773941421.931295:0:12130:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b7400. 00000040:00000010:1.0:1773941421.931301:0:12130:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000100:00000001:2.0:1773941421.931303:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000010:1.0:1773941421.931308:0:12130:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941421.931314:0:12130:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b6a00. 00000100:00000040:2.0:1773941421.931318:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60380 x1860112241144832/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941431 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 80000000:00000010:1.0:1773941421.931321:0:12130:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941421.931328:0:12130:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7c00. 00000100:00000001:2.0:1773941421.931330:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 20000000:00000010:1.0:1773941421.931334:0:12130:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6c00. 00000100:00000001:2.0:1773941421.931335:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000040:1.0:1773941421.931342:0:12130:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941421.931345:0:12130:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941421.931350:0:12130:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b7400. 00000040:00000010:1.0:1773941421.931353:0:12130:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941421.931356:0:12130:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941421.931360:0:12130:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b6a00. 80000000:00000010:1.0:1773941421.931362:0:12130:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941421.931366:0:12130:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7c00. 20000000:00000010:1.0:1773941421.931369:0:12130:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6c00. 00000020:00000010:1.0:1773941421.931371:0:12130:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b6400. 10000000:00000010:1.0:1773941421.931373:0:12130:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0fc0. 10000000:01000000:1.0:1773941421.931376:0:12130:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941421.931378:0:12130:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941421.931380:0:12130:0:(mgc_request.c:233:do_config_log_add()) MGC192.168.204.143@tcp: failed processing log, type 1: rc = -107 10000000:00000001:1.0:1773941421.935579:0:12130:0:(mgc_request.c:236:do_config_log_add()) Process leaving (rc=18446615823919920640 : -128249789630976 : ffff8b5b84d8b600) 10000000:00000001:1.0:1773941421.935590:0:12130:0:(mgc_request.c:159:config_log_find()) Process entered 10000000:00000001:1.0:1773941421.935593:0:12130:0:(mgc_request.c:177:config_log_find()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941421.935596:0:12130:0:(mgc_request.c:191:do_config_log_add()) Process entered 10000000:01000000:1.0:1773941421.935597:0:12130:0:(mgc_request.c:194:do_config_log_add()) do adding config log params-ffff8b5ba0b09800 10000000:00000010:1.0:1773941421.935607:0:12130:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 279 at ffff8b5b850b6400. 10000000:01000000:1.0:1773941421.935611:0:12130:0:(mgc_request.c:68:mgc_name2resid()) log params to resid 0x736d61726170/0x3 (params) 00000020:00000040:1.0:1773941421.935616:0:12130:0:(genops.c:887:class_export_get()) GET export ffff8b5ba0b0c000 refcount=4 10000000:00000001:1.0:1773941421.935618:0:12130:0:(mgc_request.c:236:do_config_log_add()) Process leaving (rc=18446615823923241984 : -128249786309632 : ffff8b5b850b6400) 10000000:00000001:1.0:1773941421.935633:0:12130:0:(mgc_request.c:191:do_config_log_add()) Process entered 10000000:01000000:1.0:1773941421.935634:0:12130:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-client-ffff8b5ba0b09800 10000000:00000010:1.0:1773941421.935637:0:12130:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 286 at ffff8b5b850b6c00. 10000000:01000000:1.0:1773941421.935639:0:12130:0:(mgc_request.c:68:mgc_name2resid()) log lustre-client to resid 0x65727473756c/0x0 (lustre) 00000020:00000040:1.0:1773941421.935642:0:12130:0:(genops.c:887:class_export_get()) GET export ffff8b5ba0b0c000 refcount=5 10000000:00000001:1.0:1773941421.935643:0:12130:0:(mgc_request.c:236:do_config_log_add()) Process leaving (rc=18446615823923244032 : -128249786307584 : ffff8b5b850b6c00) 10000000:00000001:1.0:1773941421.935646:0:12130:0:(mgc_request.c:191:do_config_log_add()) Process entered 10000000:01000000:1.0:1773941421.935648:0:12130:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-cliir-ffff8b5ba0b09800 10000000:00000010:1.0:1773941421.935650:0:12130:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 285 at ffff8b5b850b7c00. 10000000:01000000:1.0:1773941421.935652:0:12130:0:(mgc_request.c:68:mgc_name2resid()) log lustre-cliir to resid 0x65727473756c/0x2 (lustre) 00000020:00000040:1.0:1773941421.935655:0:12130:0:(genops.c:887:class_export_get()) GET export ffff8b5ba0b0c000 refcount=6 10000000:00000001:1.0:1773941421.935656:0:12130:0:(mgc_request.c:236:do_config_log_add()) Process leaving (rc=18446615823923248128 : -128249786303488 : ffff8b5b850b7c00) 10000000:00000001:1.0:1773941421.935659:0:12130:0:(mgc_request.c:441:config_log_add()) Process leaving (rc=18446615823923244032 : -128249786307584 : ffff8b5b850b6c00) 00000800:00000001:3.0:1773941421.935891:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941421.935906:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941421.935909:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941421.935911:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941421.935919:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941421.935924:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941421.935932:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941421.935938:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941421.935943:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941421.935948:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321603400 00000400:00000200:3.0:1773941421.935963:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941421.935971:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xad5 [1] + 0 00000800:00000001:3.0:1773941421.935980:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941421.935989:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941421.935992:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941421.935997:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941421.936004:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941421.936010:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60380 x1860112241144832/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941431 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941421.936020:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60380 x1860112241144832/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941431 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941421.936034:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941421.936055:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941421.936058:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.936067:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941421.936070:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941421.936078:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941421.936080:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941421.936083:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941421.936087:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941421.936089:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941421.936091:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941421.936096:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60380 x1860112241144832/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941431 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941421.936106:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60380 x1860112241144832/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941431 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941421.936112:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941421.936121:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.936124:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941421.936127:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941421.936130:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941421.936133:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941421.936142:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941421.936151:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=0 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941421.936155:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941421.936157:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941421.936158:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941421.936161:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941421.936163:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.936166:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941421.936170:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60380 x1860112241144832/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941431 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941421.936175:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941421.936177:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941421.936179:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.936181:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941421.936188:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941421.936190:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941421.936192:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941421.936194:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941421.936197:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941421.936198:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941421.936200:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941421.936203:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941421 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941421.936205:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941421.936212:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941421.936214:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941421.936217:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941421.936218:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.936220:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941421.936222:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941421.936224:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941421.936230:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941421.936234:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941421.936237:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941421.936239:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941421.936485:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941421.936489:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941421.936494:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60380 x1860112241144832/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941431 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941421.936502:0:12148:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:0.0:1773941421.936502:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241144832:192.168.204.143@tcp:250:mount.lustre.0 00000100:00000001:0.0:1773941421.936519:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941421.936521:0:12148:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:0.0:1773941421.936524:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1773941421.936527:0:12148:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:0.0:1773941421.936533:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60380 x1860112241144832/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941431 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941421.936534:0:12148:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941421.936538:0:12148:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941421.936545:0:12148:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:0.0:1773941421.936551:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:3.0:1773941421.936552:0:12148:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:0.0:1773941421.936558:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941421.936559:0:12148:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941421.936565:0:12148:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:0.0:1773941421.936565:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0000. 00000001:00000001:3.0:1773941421.936571:0:12148:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:0.0:1773941421.936574:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:3.0:1773941421.936575:0:12148:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941421.936579:0:2351:0:(genops.c:1236:class_import_put()) Process entered 10000000:00000001:3.0:1773941421.936580:0:12148:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941421.936584:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941421.936585:0:12148:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:0.0:1773941421.936593:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:3.0:1773941421.936594:0:12148:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:0.0:1773941421.936597:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737000. 00000100:00000001:3.0:1773941421.936599:0:12148:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:0.0:1773941421.936602:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1773941421.936604:0:12148:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:0.0:1773941421.936605:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941421.936610:0:12148:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941421.936610:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60380. 00000100:00080000:3.0:1773941421.936619:0:12148:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941421.936619:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941421.936622:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941421.936628:0:12148:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:0.0:1773941421.936628:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941421.936637:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:01000000:3.0:1773941421.936638:0:12148:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:0.0:1773941421.936639:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:3.0:1773941421.936643:0:12148:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:0.0:1773941421.936644:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941421.936648:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941421.936667:0:12148:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941421.936670:0:12148:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941421.936673:0:12148:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941421.936678:0:12148:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:3.0:1773941421.938928:0:12148:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941421.938934:0:12148:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941421.938938:0:12148:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941421.938941:0:12148:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941421.938942:0:12148:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941421.941742:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5ba0d64200. 00010000:00000010:1.1:1773941421.941751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5bc2150600. 10000000:00000001:2.0:1773941422.495815:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941422.495823:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:2.0:1773941422.495828:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941422.495838:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941422.495841:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941422.495842:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941422.495848:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941422.495849:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941422.495852:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:2.0:1773941422.495855:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941422.495857:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941422.495865:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021c380. 00000020:00000040:2.0:1773941422.495869:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941422.495872:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941422.495875:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941422.495879:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941422.495885:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b84feee00. 00000100:00000001:2.0:1773941422.495891:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941422.495894:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941422.495896:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941422.495902:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5ba0d64d00. 00010000:00000001:2.0:1773941422.495906:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941422.495909:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5bc2151e00. 00000020:00000001:2.0:1773941422.495912:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941422.495914:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5bc2151e00 with handle 0xa33a5be14eae1c52 to hash 00000020:00000001:2.0:1773941422.495915:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941422.495917:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615824947289600 : -128248762262016 : ffff8b5bc2151e00) 00010000:00000001:2.0:1773941422.495920:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615824947289600 : -128248762262016 : ffff8b5bc2151e00) 00010000:00000001:2.0:1773941422.495923:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941422.495927:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941422.495930:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c52 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941422.495956:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c52 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941422.495968:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c52 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941422.495975:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941422.495977:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941422.495982:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b54480. 00000100:00000001:2.0:1773941422.495984:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387359872 : -128249322191744 : ffff8b5ba0b54480) 00000100:00000001:2.0:1773941422.495986:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941422.495987:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109257600 : -128249600294016 : ffff8b5b9021c380) 00000100:00000001:2.0:1773941422.495990:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941422.495994:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941422.495996:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941422.495997:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941422.496005:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021c380 x1860112241144960/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941422.496011:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941422.496012:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941422.496015:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941422.496017:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941422.496019:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941422.496021:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021c380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241144960:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941422.496025:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941422.496029:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941422.496030:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941422.496032:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941422.496035:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce4400. 02000000:00000001:2.0:1773941422.496037:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941422.496042:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941422.496049:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9840. 00000400:00000010:2.0:1773941422.496054:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34ee0. 00000100:00000200:2.0:1773941422.496059:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241144960, portal 25 00000100:00000001:2.0:1773941422.496060:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941422.496061:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109257600 : -128249600294016 : ffff8b5b9021c380) 00000100:00000040:2.0:1773941422.496065:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021c380 x1860112241144960/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941438 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941422.496070:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941422.496072:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941422.496074:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34a18. 00000100:00000200:2.0:1773941422.496075:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241144960, offset 0 00000400:00000200:2.0:1773941422.496080:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941422.496085:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941422.496090:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941422.496093:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941422.496096:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941422.496099:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941422.496101:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941422.496103:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[6, 5] 00000400:00000200:2.0:1773941422.496105:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:2.0:1773941422.496107:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [112:112:256:4294967295] 192.168.204.153@tcp LPNI seq info [7:7:8:4294967295] 00000400:00000200:2.0:1773941422.496110:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941422.496115:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:2.0:1773941422.496119:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:2.0:1773941422.496123:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0200. 00000800:00000200:2.0:1773941422.496127:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:2.0:1773941422.496132:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:2.0:1773941422.496136:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0200 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941422.496149:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941422.496152:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941422.496154:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294862859 00000100:00000001:2.0:1773941422.496163:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294862294 00000100:00000001:2.0:1773941422.496166:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294862859 00000100:00000001:2.0:1773941422.496169:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294862294 00000100:00000001:2.0:1773941422.496172:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941422.496174:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941422.496176:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941422.496177:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941422.496179:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b54480 going to sleep for 11 seconds 00000100:00000001:2.0:1773941422.496181:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941422.496182:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941422.496265:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941422.496269:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0200. 00000400:00000200:3.0:1773941422.496275:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941422.496280:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941422.496284:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34a18 00000400:00000010:3.0:1773941422.496286:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34a18. 00000100:00000001:3.0:1773941422.496289:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941422.496292:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021c380 x1860112241144960/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941438 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941422.496300:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941422.496303:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b9021c380 x1860112241144960/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941438 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941422.496307:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941422.496309:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941422.497135:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941422.497145:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941422.497148:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941422.497151:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941422.497156:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941422.497160:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941422.497166:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941422.497168:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941422.497211:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941422.497219:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321603480 00000400:00000200:3.0:1773941422.497222:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941422.497225:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xadd [1] + 224 00000800:00000001:3.0:1773941422.497230:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941422.497257:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941422.497259:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941422.497262:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941422.497265:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941422.497268:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021c380 x1860112241144960/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941438 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941422.497275:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b9021c380 x1860112241144960/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941438 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941422.497286:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941422.497290:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941422.497293:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941422.497315:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941422.497322:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941422.497323:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941422.497325:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9840. 00000400:00000200:2.0:1773941422.497329:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34ee0 00000400:00000010:2.0:1773941422.497331:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34ee0. 00000100:00000001:2.0:1773941422.497333:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941422.497339:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021c380 x1860112241144960/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941438 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941422.497349:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021c380 x1860112241144960/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941438 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941422.497356:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941422.497358:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941422.497360:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941422.497363:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941422.497365:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941422.497366:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941422.497369:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941422.497373:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=1 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941422.497377:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941422.497379:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941422.497383:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:2.0:1773941422.500265:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941422.500281:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941422.500286:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941422.500291:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941422.500297:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941422.503514:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941422.503517:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941422.503518:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941422.503520:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941422.503526:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b9021c380 x1860112241144960/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941438 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941422.503535:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941422.503537:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941422.503539:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941422.503541:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941422.503543:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941422.503545:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941422.503546:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941422.503547:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941422.503547:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941422.503548:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941422.503549:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941422.503551:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941422.503554:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941422.503557:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941422.503559:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 192 00000400:00000200:2.0:1773941422.503568:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941422.503573:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941422.503577:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941422.503580:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941422.503582:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941422.503584:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:2.0:1773941422.503587:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941422.503599:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941422.503601:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941422.503603:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941422.503605:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941422.503606:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:2.0:1773941422.503608:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941422.503610:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941422.503611:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941422.503612:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:2.0:1773941422.503614:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:2.0:1773941422.503616:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:2.0:1773941422.503618:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941422.503620:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941422.503622:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941422.503624:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941422.503625:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941422.503627:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941422.503632:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021e300. 00000020:00000040:2.0:1773941422.503634:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:2.0:1773941422.503637:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941422.503638:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941422.503640:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941422.503642:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941422.503645:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941422.503650:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0ce5800. 00000100:00000001:2.0:1773941422.503655:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941422.503659:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b9021e300 x1860112241145088/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941422.503666:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941422.503669:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941422.503673:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083d4724a] to pc [ptlrpcd_rcv+-1] req@ffff8b5b9021e300 x1860112241145088/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941422.503695:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941422.503698:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941422.503699:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941422.503702:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941422.503717:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941422.503719:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941422.503721:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b54480 going to sleep for 1 seconds 00000100:00000001:2.0:1773941422.503723:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941422.503726:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941422.503732:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021c380 x1860112241144960/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941438 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941422.503738:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941422.503739:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941422.503744:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941422.503746:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941422.503748:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941422.503757:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1773941422.503758:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021c380 x1860112241144960/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941438 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941422.503765:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021c380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241144960:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941422.503768:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941422.503770:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941422.503773:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941422.503776:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1773941422.503777:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941422.503790:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021c380 x1860112241144960/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941438 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941422.503798:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021e300 x1860112241145088/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941422.503803:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941422.503813:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b54480. 00000100:00000001:2.0:1773941422.503824:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941422.503829:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941422.503830:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1773941422.503834:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:0.0:1773941422.503837:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941422.503842:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:2.0:1773941422.503843:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941422.503847:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941422.503850:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5bc2151e00 refcount=4 02000000:00000001:0.0:1773941422.503850:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1773941422.503865:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824947289600 : -128248762262016 : ffff8b5bc2151e00) 02000000:00000001:0.0:1773941422.503865:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941422.503878:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615824947289600 : -128248762262016 : ffff8b5bc2151e00) 00000100:00100000:0.0:1773941422.503882:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241145088:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00010000:2.0:1773941422.503885:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c52 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941422.503887:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941422.503894:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:2.0:1773941422.503896:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:0.0:1773941422.503897:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941422.503901:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c52 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941422.503901:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:2.0:1773941422.503911:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:2.0:1773941422.503914:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c52 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941422.503918:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000010:0.0:1773941422.503919:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0000. 00010000:00000001:2.0:1773941422.503921:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 02000000:00000001:0.0:1773941422.503925:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941422.503927:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c52 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941422.503936:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:0.0:1773941422.503938:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:2.0:1773941422.503941:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941422.503947:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000010:0.0:1773941422.503947:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00010000:2.0:1773941422.503954:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c52 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941422.503960:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00010000:00010000:2.0:1773941422.503964:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5bc2151e00 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:2.0:1773941422.503969:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000200:0.0:1773941422.503971:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241145088, portal 25 10000000:00010000:2.0:1773941422.503973:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c52 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941422.503975:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941422.503979:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109265664 : -128249600285952 : ffff8b5b9021e300) 00000100:00000040:0.0:1773941422.503992:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021e300 x1860112241145088/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941432 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941422.504031:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941422.504033:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:0.0:1773941422.504033:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1773941422.504038:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941422.504040:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5bc2151e00 refcount=5 00000100:00000040:0.0:1773941422.504043:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000020:00000001:2.0:1773941422.504047:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824947289600 : -128248762262016 : ffff8b5bc2151e00) 00000400:00000010:0.0:1773941422.504050:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c110. 00010000:00000001:2.0:1773941422.504052:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615824947289600 : -128248762262016 : ffff8b5bc2151e00) 00010000:00000001:2.0:1773941422.504057:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000200:0.0:1773941422.504058:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241145088, offset 0 00010000:00010000:2.0:1773941422.504063:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c52 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:2.0:1773941422.504076:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941422.504076:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 10000000:00010000:2.0:1773941422.504087:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c52 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1773941422.504092:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:2.0:1773941422.504094:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941422.504099:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941422.504099:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:2.0:1773941422.504107:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941422.504110:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1773941422.504114:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:0.0:1773941422.504116:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:2.0:1773941422.504118:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941422.504123:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941422.504126:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941422.504126:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:2.0:1773941422.504133:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5bc2151e00 with handle 0xa33a5be14eae1c52 from hash 00000400:00000200:0.0:1773941422.504136:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:2.0:1773941422.504139:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:0.0:1773941422.504143:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941422.504144:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941422.504147:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941422.504152:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941422.504156:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:0.0:1773941422.504159:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941422.504160:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:2.0:1773941422.504165:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941422.504167:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[6, 7] 00010000:00000001:2.0:1773941422.504168:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941422.504172:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941422.504176:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 10000000:00000001:2.0:1773941422.504186:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941422.504186:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [113:113:256:4294967295] 192.168.204.143@tcp LPNI seq info [8:8:8:4294967295] 00010000:00010000:2.0:1773941422.504197:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c52 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941422.504200:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:2.0:1773941422.504211:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941422.504213:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941422.504214:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941422.504217:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941422.504222:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941422.504226:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:0.0:1773941422.504227:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:2.0:1773941422.504228:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941422.504248:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c52 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941422.504257:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000040:2.0:1773941422.504261:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5ba0d64d00 count: 0 00000800:00000010:0.0:1773941422.504266:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00010000:00000001:2.0:1773941422.504274:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941422.504276:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941422.504285:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000800:00000200:0.0:1773941422.504285:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000100:00000040:2.0:1773941422.504294:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021c380 x1860112241144960/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941438 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1773941422.504306:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000100:00000001:2.0:1773941422.504311:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941422.504315:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000200:0.0:1773941422.504315:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 02000000:00000010:2.0:1773941422.504318:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce4400. 02000000:00000001:2.0:1773941422.504322:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941422.504324:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941422.504325:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941422.504327:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941422.504329:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b84feee00. 02000000:00000001:2.0:1773941422.504332:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941422.504333:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941422.504336:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941422.504337:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021c380. 00000100:00000001:2.0:1773941422.504348:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941422.504350:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941422.504351:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:2.0:1773941422.504356:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941422.504358:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294862867 10000000:01000000:2.0:1773941422.504362:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941422.504366:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:0.0:1773941422.504369:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294862859 10000000:00000010:2.0:1773941422.504377:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b845add00. 00000100:00000001:0.0:1773941422.504380:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294862867 00000020:00000010:2.0:1773941422.504383:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84feee00. 00000100:00000001:0.0:1773941422.504387:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294862859 00000020:00000010:2.0:1773941422.504388:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b84fef200. 00000040:00000010:2.0:1773941422.504397:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b84fee000. 00000100:00000001:0.0:1773941422.504397:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1773941422.504412:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b84e45800. 00000100:00000001:0.0:1773941422.504414:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941422.504417:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b84fee800. 00000100:00000001:0.0:1773941422.504419:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941422.504423:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 80000000:00000010:2.0:1773941422.504424:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba21bdf60. 00040000:00000010:2.0:1773941422.504435:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b84fee400. 00000100:00000001:0.0:1773941422.504437:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 20000000:00000010:2.0:1773941422.504439:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b90e7fe00. 00000100:00000001:0.0:1773941422.504441:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 10000000:00000040:2.0:1773941422.504446:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000100:00000001:0.0:1773941422.504447:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:3.0:1773941422.504448:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1773941422.504451:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000001:2.0:1773941422.504453:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941422.504456:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1773941422.504458:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000100:00000001:0.0:1773941422.504463:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941422.504467:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b84fef200. 00000400:00000200:3.0:1773941422.504475:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000040:00000010:2.0:1773941422.504478:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b84fee000. 00000020:00000010:2.0:1773941422.504483:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b84e45800. 00000400:00000200:3.0:1773941422.504485:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000020:00000010:2.0:1773941422.504489:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b84fee800. 00000400:00000200:3.0:1773941422.504494:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c110 80000000:00000010:2.0:1773941422.504495:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba21bdf60. 00000400:00000010:3.0:1773941422.504503:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c110. 00040000:00000010:2.0:1773941422.504503:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b84fee400. 20000000:00000010:2.0:1773941422.504505:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b90e7fe00. 00000020:00000010:2.0:1773941422.504509:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84feee00. 00000100:00000001:3.0:1773941422.504511:0:2347:0:(events.c:34:request_out_callback()) Process entered 10000000:00000010:2.0:1773941422.504512:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b845add00. 10000000:01000000:2.0:1773941422.504516:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941422.504519:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000200:3.0:1773941422.504521:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021e300 x1860112241145088/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941432 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00020000:2.0:1773941422.504522:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000100:00000001:3.0:1773941422.504598:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941422.504603:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021e300 x1860112241145088/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941432 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941422.504611:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941422.504613:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941422.507598:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941422.507609:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941422.507612:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941422.507622:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941422.507629:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941422.507634:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941422.507737:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941422.507740:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941422.507745:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941422.507749:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321603500 00000400:00000200:3.0:1773941422.507753:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941422.507757:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xaed [1] + 0 00000800:00000001:3.0:1773941422.507764:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941422.507771:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941422.507773:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941422.507775:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941422.507779:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941422.507785:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021e300 x1860112241145088/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941432 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941422.507793:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b9021e300 x1860112241145088/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941432 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941422.507808:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941422.507813:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941422.507815:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941422.507846:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941422.507849:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941422.507859:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941422.507861:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941422.507863:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941422.507872:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941422.507873:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941422.507875:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941422.507879:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021e300 x1860112241145088/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941432 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941422.507886:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021e300 x1860112241145088/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941432 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941422.507891:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941422.507892:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941422.507893:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941422.507896:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941422.507897:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941422.507899:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941422.507901:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941422.507906:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=1 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941422.507909:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941422.507912:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941422.507917:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941422.507919:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941422.507921:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941422.507922:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941422.507926:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021e300 x1860112241145088/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941432 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941422.507931:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941422.507932:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941422.507934:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941422.507935:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941422.507940:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 10000000:00000001:2.0:1773941422.507942:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 00000020:00000001:0.0:1773941422.507942:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941422.507946:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 10000000:00000001:2.0:1773941422.507947:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 00000020:00000001:0.0:1773941422.507948:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 10000000:00000040:2.0:1773941422.507950:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 00000020:00000040:0.0:1773941422.507951:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 10000000:00000001:2.0:1773941422.507956:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000020:00000001:0.0:1773941422.507956:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941422.507960:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941422.507967:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941422 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941422.507969:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941422.507974:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941422.507976:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941422.507978:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941422.507979:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941422.507980:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941422.507983:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941422.507984:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941422.507986:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941422.507988:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941422.507990:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941422.507992:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941422.508195:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941422.508198:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941422.508203:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021e300 x1860112241145088/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941432 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941422.508209:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241145088:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941422.508214:0:12149:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:0.0:1773941422.508214:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941422.508218:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941422.508222:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021e300 x1860112241145088/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941432 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1773941422.508225:0:12149:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:0.0:1773941422.508230:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:2.0:1773941422.508232:0:12149:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:0.0:1773941422.508241:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941422.508244:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0000. 02000000:00000001:0.0:1773941422.508247:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1773941422.508249:0:12149:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000001:0.0:1773941422.508250:0:2351:0:(genops.c:1236:class_import_put()) Process entered 10000000:01000000:2.0:1773941422.508253:0:12149:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000040:0.0:1773941422.508253:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941422.508258:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:2.0:1773941422.508259:0:12149:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:0.0:1773941422.508260:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0ce5800. 00000001:00000001:2.0:1773941422.508264:0:12149:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:0.0:1773941422.508264:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941422.508271:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941422.508273:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021e300. 00000100:00000001:0.0:1773941422.508276:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:2.0:1773941422.508278:0:12149:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941422.508278:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:2.0:1773941422.508282:0:12149:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:0.0:1773941422.508283:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:2.0:1773941422.508285:0:12149:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941422.508287:0:12149:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941422.508288:0:12149:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941422.508290:0:12149:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941422.508290:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1773941422.508295:0:12149:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941422.508296:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941422.508297:0:12149:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941422.508299:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941422.508300:0:12149:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941422.508302:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941422.508303:0:12149:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941422.508306:0:12149:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941422.508307:0:12149:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941422.508309:0:12149:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941422.508310:0:12149:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:2.0:1773941422.508325:0:12149:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941422.508327:0:12149:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941422.508328:0:12149:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941422.508331:0:12149:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941422.512024:0:12149:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941422.512030:0:12149:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941422.512032:0:12149:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941422.512035:0:12149:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941422.512036:0:12149:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941422.513735:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5ba0d64d00. 00010000:00000010:2.1:1773941422.513744:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5bc2151e00. 00000020:00000001:2.0:1773941422.943811:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941422.943817:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941422.943820:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1773941423.967864:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941423.967872:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941423.967877:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941424.031798:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941424.031806:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941424.031810:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941424.031812:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941424.031813:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941424.031815:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941424.031817:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941424.031819:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941424.031822:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941424.031824:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941424.031826:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941424.031836:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62d80. 00000020:00000040:1.0:1773941424.031841:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941424.031846:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941424.031849:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941424.031854:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941424.031861:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7400. 00000100:00000001:1.0:1773941424.031868:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941424.031871:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941424.031874:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941424.031881:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec200. 00010000:00000001:1.0:1773941424.031901:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941424.031904:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f000. 00000020:00000001:1.0:1773941424.031908:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941424.031911:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f000 with handle 0xa33a5be14eae1c59 to hash 00000020:00000001:1.0:1773941424.031913:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941424.031915:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941424.031917:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941424.031921:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941424.031923:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941424.031928:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c59 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941424.031938:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c59 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941424.031949:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c59 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941424.031955:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941424.031957:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941424.031964:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6c00. 00000100:00000001:1.0:1773941424.031966:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258984960 : -128249450566656 : ffff8b5b990e6c00) 00000100:00000001:1.0:1773941424.031968:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941424.031970:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121081216 : -128249588470400 : ffff8b5b90d62d80) 00000100:00000001:1.0:1773941424.031974:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941424.031978:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941424.031980:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941424.031982:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941424.031992:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62d80 x1860112241145216/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941424.032001:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941424.032003:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941424.032007:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941424.032014:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941424.032017:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941424.032021:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241145216:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941424.032025:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941424.032031:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941424.032033:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941424.032036:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941424.032040:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735c00. 02000000:00000001:1.0:1773941424.032042:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941424.032049:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941424.032054:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941424.032061:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941424.032068:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241145216, portal 25 00000100:00000001:1.0:1773941424.032070:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941424.032071:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121081216 : -128249588470400 : ffff8b5b90d62d80) 00000100:00000040:1.0:1773941424.032077:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62d80 x1860112241145216/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941440 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941424.032085:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941424.032088:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941424.032091:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1dd0. 00000100:00000200:1.0:1773941424.032094:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241145216, offset 0 00000400:00000200:1.0:1773941424.032100:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941424.032107:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941424.032114:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941424.032119:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941424.032123:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941424.032127:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941424.032130:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941424.032133:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[8, 7] 00000400:00000200:1.0:1773941424.032137:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941424.032140:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [114:114:256:4294967295] 192.168.204.153@tcp LPNI seq info [9:9:8:4294967295] 00000400:00000200:1.0:1773941424.032145:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941424.032161:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941424.032167:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941424.032172:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b200. 00000800:00000200:1.0:1773941424.032178:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941424.032185:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941424.032189:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b200 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941424.032232:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941424.032238:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941424.032241:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294864395 00000100:00000001:1.0:1773941424.032248:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294862867 00000100:00000001:1.0:1773941424.032253:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294864395 00000100:00000001:1.0:1773941424.032257:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294862867 00000100:00000001:1.0:1773941424.032260:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941424.032262:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941424.032264:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941424.032266:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941424.032269:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6c00 going to sleep for 11 seconds 00000100:00000001:1.0:1773941424.032272:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941424.032275:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941424.032315:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941424.032320:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b200. 00000400:00000200:3.0:1773941424.032324:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941424.032328:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941424.032332:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1dd0 00000400:00000010:3.0:1773941424.032334:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1dd0. 00000100:00000001:3.0:1773941424.032337:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941424.032340:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62d80 x1860112241145216/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941440 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941424.032347:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941424.032350:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d62d80 x1860112241145216/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941440 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941424.032359:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941424.032360:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941424.033376:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941424.033386:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941424.033389:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941424.033391:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941424.033396:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941424.033400:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941424.033406:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941424.033448:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941424.033454:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941424.033458:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321603580 00000400:00000200:3.0:1773941424.033462:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941424.033467:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xaf5 [1] + 224 00000800:00000001:3.0:1773941424.033474:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941424.033481:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941424.033483:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941424.033487:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941424.033490:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941424.033495:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62d80 x1860112241145216/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941440 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941424.033506:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d62d80 x1860112241145216/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941440 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941424.033522:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941424.033528:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941424.033531:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941424.033559:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941424.033565:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941424.033567:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941424.033570:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941424.033573:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941424.033575:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941424.033577:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941424.033582:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62d80 x1860112241145216/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941440 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941424.033592:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62d80 x1860112241145216/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941440 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941424.033598:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941424.033606:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941424.033608:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941424.033611:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941424.033613:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941424.033615:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941424.033618:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941424.033623:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=3 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941424.033627:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941424.033630:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941424.033633:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941424.036518:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941424.036526:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941424.036530:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941424.036533:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941424.036536:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941424.039904:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941424.039907:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941424.039908:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941424.039910:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941424.039915:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d62d80 x1860112241145216/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941440 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941424.039924:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941424.039925:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941424.039926:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941424.039929:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941424.039931:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941424.039933:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941424.039934:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941424.039935:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941424.039936:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941424.039937:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941424.039938:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941424.039940:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941424.039941:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941424.039956:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941424.039958:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 192 00000400:00000200:1.0:1773941424.039964:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941424.039967:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941424.039970:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941424.039972:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941424.039974:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941424.039975:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941424.039978:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941424.039979:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941424.039981:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941424.039983:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941424.039985:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941424.039986:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941424.039988:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941424.039989:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941424.039990:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941424.039992:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941424.039994:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941424.039996:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941424.039998:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941424.039999:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941424.040002:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941424.040003:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941424.040005:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941424.040006:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941424.040010:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63480. 00000020:00000040:1.0:1773941424.040012:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941424.040014:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941424.040015:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941424.040017:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941424.040019:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941424.040024:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941424.040029:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0736c00. 00000100:00000001:1.0:1773941424.040034:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941424.040038:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63480 x1860112241145344/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941424.040045:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941424.040048:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941424.040051:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000834a8cee] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63480 x1860112241145344/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941424.040068:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941424.040071:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941424.040072:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941424.040075:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941424.040078:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941424.040079:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941424.040081:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6c00 going to sleep for 1 seconds 00000100:00000001:1.0:1773941424.040083:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941424.040085:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941424.040090:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62d80 x1860112241145216/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941440 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941424.040094:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941424.040097:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941424.040098:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941424.040100:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941424.040103:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62d80 x1860112241145216/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941440 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941424.040108:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241145216:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941424.040111:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941424.040113:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941424.040114:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941424.040114:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941424.040119:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941424.040123:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941424.040123:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941424.040132:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62d80 x1860112241145216/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941440 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941424.040134:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63480 x1860112241145344/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941424.040139:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941424.040143:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6c00. 00000100:00000001:1.0:1773941424.040149:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941424.040151:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941424.040152:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941424.040155:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941424.040156:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941424.040160:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941424.040163:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:0.0:1773941424.040163:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1773941424.040167:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=4 02000000:00000001:0.0:1773941424.040167:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1773941424.040169:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 02000000:00000001:0.0:1773941424.040174:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941424.040175:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00010000:1.0:1773941424.040182:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c59 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1773941424.040184:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241145344:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941424.040193:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941424.040194:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00010000:1.0:1773941424.040198:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c59 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941424.040199:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941424.040202:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941424.040219:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941424.040220:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941424.040223:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c59 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:0.0:1773941424.040226:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c3000. 02000000:00000001:0.0:1773941424.040231:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941424.040232:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941424.040235:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941424.040237:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c59 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941424.040241:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941424.040245:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941424.040249:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1773941424.040249:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00000001:1.0:1773941424.040253:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000010:0.0:1773941424.040257:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00010000:00010000:1.0:1773941424.040260:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c59 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:0.0:1773941424.040265:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241145344, portal 25 00010000:00010000:1.0:1773941424.040266:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f000 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941424.040270:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:0.0:1773941424.040272:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 10000000:00010000:1.0:1773941424.040273:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c59 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941424.040275:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083008 : -128249588468608 : ffff8b5b90d63480) 00010000:00000001:1.0:1773941424.040279:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941424.040280:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941424.040281:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941424.040281:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=5 00000020:00000001:1.0:1773941424.040283:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941424.040288:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941424.040290:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941424.040296:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c59 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941424.040296:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63480 x1860112241145344/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941434 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941424.040306:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941424.040310:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c59 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941424.040313:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941424.040318:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941424.040321:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c330. 10000000:01000000:1.0:1773941424.040323:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000200:0.0:1773941424.040326:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241145344, offset 0 10000000:00000040:1.0:1773941424.040327:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941424.040330:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941424.040334:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:0.0:1773941424.040334:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941424.040337:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941424.040340:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:0.0:1773941424.040340:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941424.040344:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941424.040348:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:0.0:1773941424.040348:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:1.0:1773941424.040352:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941424.040353:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:1.0:1773941424.040357:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f000 with handle 0xa33a5be14eae1c59 from hash 00000400:00000200:0.0:1773941424.040358:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941424.040363:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:0.0:1773941424.040365:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941424.040367:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941424.040368:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[8, 9] 00010000:00000001:1.0:1773941424.040374:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941424.040374:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941424.040378:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:0.0:1773941424.040380:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [115:115:256:4294967295] 192.168.204.143@tcp LPNI seq info [10:10:8:4294967295] 00010000:00000001:1.0:1773941424.040381:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941424.040385:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941424.040387:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:0.0:1773941424.040388:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941424.040392:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941424.040394:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941424.040398:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941424.040401:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 10000000:00000001:1.0:1773941424.040402:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941424.040408:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c59 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941424.040413:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941424.040420:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000010:0.0:1773941424.040421:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00010000:00000001:1.0:1773941424.040422:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941424.040426:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941424.040427:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941424.040429:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941424.040431:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941424.040433:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:0.0:1773941424.040433:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00010000:1.0:1773941424.040437:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c59 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941424.040444:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000040:1.0:1773941424.040449:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec200 count: 0 00000800:00000200:0.0:1773941424.040449:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941424.040466:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941424.040467:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941424.040469:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:0.0:1773941424.040473:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941424.040476:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941424.040477:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294864403 00000100:00000040:1.0:1773941424.040484:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62d80 x1860112241145216/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941440 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941424.040486:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294864395 00000100:00000001:0.0:1773941424.040494:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294864403 00000100:00000001:1.0:1773941424.040496:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941424.040500:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294864395 02000000:00000001:1.0:1773941424.040501:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1773941424.040504:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1773941424.040506:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735c00. 02000000:00000001:1.0:1773941424.040510:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1773941424.040511:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941424.040513:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941424.040515:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941424.040517:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941424.040518:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1773941424.040523:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941424.040526:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000010:1.0:1773941424.040527:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7400. 00000100:00000001:0.0:1773941424.040529:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1773941424.040531:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941424.040532:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941424.040533:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941424.040534:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941424.040541:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941424.040545:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941424.040547:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62d80. 00000100:00000001:1.0:1773941424.040551:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941424.040553:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941424.040555:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941424.040558:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941424.040561:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941424.040564:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0880. 00000020:00000010:1.0:1773941424.040568:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7400. 00000020:00000010:1.0:1773941424.040572:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b7000. 00000040:00000010:1.0:1773941424.040576:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6e00. 00000800:00000200:3.0:1773941424.040578:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000020:00000010:1.0:1773941424.040581:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000800:00000010:3.0:1773941424.040584:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000020:00000010:1.0:1773941424.040587:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b6800. 00000400:00000200:3.0:1773941424.040591:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 80000000:00000010:1.0:1773941424.040594:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000400:00000200:3.0:1773941424.040600:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00040000:00000010:1.0:1773941424.040601:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7a00. 00000400:00000200:3.0:1773941424.040604:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c330 20000000:00000010:1.0:1773941424.040607:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b7800. 10000000:00000040:1.0:1773941424.040612:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000400:00000010:3.0:1773941424.040613:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c330. 10000000:00000001:1.0:1773941424.040617:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941424.040619:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941424.040625:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63480 x1860112241145344/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941434 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941424.040625:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b7000. 00000040:00000010:1.0:1773941424.040632:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6e00. 00000100:00000001:3.0:1773941424.040633:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941424.040639:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63480 x1860112241145344/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941434 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941424.040639:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941424.040646:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941424.040651:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941424.040656:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7a00. 20000000:00000010:1.0:1773941424.040658:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941424.040661:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7400. 10000000:00000010:1.0:1773941424.040663:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0880. 10000000:01000000:1.0:1773941424.040667:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941424.040669:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941424.040672:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000100:00000001:3.0:1773941424.040691:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941424.040693:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000001:1.0:1773941424.042991:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941424.042994:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941424.042996:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941424.043009:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941424.044338:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941424.044349:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941424.044351:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941424.044352:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941424.044358:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941424.044370:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941424.044429:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941424.044435:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941424.044440:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941424.044443:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321603600 00000400:00000200:3.0:1773941424.044446:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941424.044450:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xafd [1] + 0 00000800:00000001:3.0:1773941424.044455:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941424.044471:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941424.044473:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941424.044476:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941424.044481:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941424.044486:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63480 x1860112241145344/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941434 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941424.044494:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63480 x1860112241145344/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941434 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941424.044507:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941424.044519:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941424.044521:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941424.044557:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941424.044562:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941424.044568:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941424.044570:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941424.044573:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941424.044578:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941424.044580:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941424.044583:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941424.044589:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63480 x1860112241145344/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941434 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941424.044601:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63480 x1860112241145344/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941434 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941424.044610:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941424.044612:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941424.044614:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941424.044617:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941424.044619:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941424.044621:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941424.044623:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941424.044629:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=3 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941424.044632:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941424.044635:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941424.044637:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941424.044639:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941424.044641:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941424.044643:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941424.044648:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63480 x1860112241145344/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941434 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941424.044668:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941424.044670:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941424.044672:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941424.044675:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941424.044682:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941424.044685:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941424.044686:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941424.044688:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941424.044691:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941424.044693:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941424.044696:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941424.044699:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941424 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941424.044730:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941424.044737:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941424.044741:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941424.044744:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941424.044746:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941424.044748:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941424.044752:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941424.044754:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941424.044757:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941424.044760:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941424.044764:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941424.044766:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941424.044998:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941424.045002:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941424.045007:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63480 x1860112241145344/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941434 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941424.045015:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241145344:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941424.045019:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941424.045021:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941424.045023:0:12151:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941424.045030:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63480 x1860112241145344/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941434 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941424.045034:0:12151:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:2.0:1773941424.045042:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:0.0:1773941424.045042:0:12151:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:2.0:1773941424.045045:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1773941424.045047:0:12151:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:2.0:1773941424.045049:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c3000. 10000000:01000000:0.0:1773941424.045050:0:12151:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941424.045053:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941424.045055:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941424.045055:0:12151:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000040:2.0:1773941424.045059:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941424.045061:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:0.0:1773941424.045061:0:12151:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941424.045063:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0736c00. 02000000:00000001:2.0:1773941424.045065:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941424.045067:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941424.045070:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63480. 00000100:00000001:2.0:1773941424.045072:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941424.045073:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:0.0:1773941424.045074:0:12151:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941424.045077:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:0.0:1773941424.045079:0:12151:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941424.045082:0:12151:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941424.045083:0:12151:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941424.045084:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000001:0.0:1773941424.045086:0:12151:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941424.045087:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941424.045091:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941424.045091:0:12151:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941424.045095:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941424.045098:0:12151:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941424.045102:0:12151:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941424.045104:0:12151:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941424.045107:0:12151:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941424.045109:0:12151:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941424.045111:0:12151:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941424.045113:0:12151:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941424.045114:0:12151:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941424.045133:0:12151:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941424.045135:0:12151:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941424.045136:0:12151:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941424.045140:0:12151:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941424.048722:0:12151:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941424.048730:0:12151:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941424.048733:0:12151:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941424.048738:0:12151:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941424.048739:0:12151:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941424.049745:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec200. 00010000:00000010:1.1:1773941424.049756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f000. 00000020:00000001:2.0:1773941424.991834:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941424.991840:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941424.991845:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941425.951761:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941425.951767:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941425.951770:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941425.951772:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941425.951773:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941425.951774:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941425.951776:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941425.951777:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941425.951779:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941425.951781:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941425.951782:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941425.951789:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62d80. 00000020:00000040:1.0:1773941425.951792:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941425.951795:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941425.951798:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941425.951803:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941425.951807:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7400. 00000100:00000001:1.0:1773941425.951812:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941425.951815:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941425.951817:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941425.951823:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec200. 00010000:00000001:1.0:1773941425.951827:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941425.951829:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f000. 00000020:00000001:1.0:1773941425.951832:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941425.951833:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f000 with handle 0xa33a5be14eae1c60 to hash 00000020:00000001:1.0:1773941425.951835:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941425.951836:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941425.951838:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941425.951840:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941425.951843:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941425.951846:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c60 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941425.951853:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c60 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941425.951869:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c60 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941425.951873:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941425.951875:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941425.951879:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6c00. 00000100:00000001:1.0:1773941425.951881:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258984960 : -128249450566656 : ffff8b5b990e6c00) 00000100:00000001:1.0:1773941425.951882:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941425.951883:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121081216 : -128249588470400 : ffff8b5b90d62d80) 00000100:00000001:1.0:1773941425.951887:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941425.951890:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941425.951892:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941425.951893:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941425.951900:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62d80 x1860112241145472/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941425.951906:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941425.951907:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941425.951909:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941425.951911:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941425.951912:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941425.951914:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241145472:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941425.951917:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941425.951921:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941425.951922:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941425.951924:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941425.951926:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735c00. 02000000:00000001:1.0:1773941425.951927:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941425.951933:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941425.951936:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941425.951941:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941425.951945:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241145472, portal 25 00000100:00000001:1.0:1773941425.951946:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941425.951950:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121081216 : -128249588470400 : ffff8b5b90d62d80) 00000100:00000040:1.0:1773941425.951954:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62d80 x1860112241145472/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941441 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941425.951959:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941425.951960:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941425.951962:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1cc0. 00000100:00000200:1.0:1773941425.951964:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241145472, offset 0 00000400:00000200:1.0:1773941425.951969:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941425.951974:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941425.951979:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941425.951983:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941425.951985:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941425.951988:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941425.951990:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941425.951992:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[10, 9] 00000400:00000200:1.0:1773941425.951995:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941425.951997:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [116:116:256:4294967295] 192.168.204.153@tcp LPNI seq info [11:11:8:4294967295] 00000400:00000200:1.0:1773941425.952000:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941425.952007:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941425.952010:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941425.952013:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b600. 00000800:00000200:1.0:1773941425.952018:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941425.952024:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941425.952027:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941425.952040:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941425.952042:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941425.952044:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294866315 00000100:00000001:1.0:1773941425.952049:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294864403 00000100:00000001:1.0:1773941425.952052:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294866315 00000100:00000001:1.0:1773941425.952055:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294864403 00000100:00000001:1.0:1773941425.952062:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941425.952063:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941425.952065:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941425.952066:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941425.952068:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6c00 going to sleep for 11 seconds 00000100:00000001:1.0:1773941425.952071:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941425.952073:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941425.952127:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941425.952131:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b600. 00000400:00000200:3.0:1773941425.952135:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941425.952139:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941425.952143:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1cc0 00000400:00000010:3.0:1773941425.952145:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1cc0. 00000100:00000001:3.0:1773941425.952148:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941425.952151:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62d80 x1860112241145472/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941441 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941425.952158:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941425.952160:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d62d80 x1860112241145472/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941441 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941425.952165:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941425.952166:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941425.952933:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941425.952940:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941425.952943:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941425.952944:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941425.952947:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941425.952950:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941425.952954:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941425.952983:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941425.952986:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941425.952989:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321603680 00000400:00000200:3.0:1773941425.952991:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941425.952994:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xb05 [1] + 224 00000800:00000001:3.0:1773941425.952999:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941425.953003:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941425.953005:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941425.953007:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941425.953010:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941425.953017:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62d80 x1860112241145472/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941441 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941425.953023:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d62d80 x1860112241145472/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941441 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941425.953034:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941425.953040:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941425.953041:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941425.953061:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941425.953066:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941425.953067:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941425.953069:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941425.953071:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941425.953073:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941425.953074:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941425.953078:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62d80 x1860112241145472/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941441 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941425.953085:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62d80 x1860112241145472/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941441 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941425.953089:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941425.953090:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941425.953092:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941425.953093:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941425.953095:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941425.953097:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941425.953098:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941425.953108:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=4 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941425.953111:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941425.953115:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941425.953118:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941425.953119:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941425.953122:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941425.953124:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941425.953126:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941425.953129:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941425.953137:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941425.953138:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941425.953139:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941425.953141:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941425.953143:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d62d80 x1860112241145472/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941441 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941425.953147:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941425.953148:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941425.953150:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941425.953151:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941425.953153:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941425.953154:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941425.953156:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941425.953156:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941425.953157:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941425.953158:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941425.953159:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941425.953161:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941425.953162:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941425.953164:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941425.953165:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 194 00000400:00000200:1.0:1773941425.953181:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941425.953187:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941425.953190:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941425.953192:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941425.953193:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941425.953194:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941425.953197:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941425.953198:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941425.953200:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941425.953202:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941425.953204:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941425.953207:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941425.953215:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941425.953217:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941425.953218:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941425.953220:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941425.953223:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941425.953226:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941425.953229:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941425.953231:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941425.953233:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941425.953236:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941425.953238:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941425.953240:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941425.953243:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941425.953246:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941425.953249:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941425.953250:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941425.953253:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941425.953254:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941425.953256:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941425.953260:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734c00. 00000100:00000001:1.0:1773941425.953262:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941425.953266:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63800 x1860112241145600/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941425.953271:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941425.953273:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941425.953275:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc928f9c] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63800 x1860112241145600/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941425.953290:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941425.953291:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941425.953293:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941425.953295:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941425.953296:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941425.953301:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941425.953302:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6c00 going to sleep for 1 seconds 00000100:00000001:1.0:1773941425.953303:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941425.953304:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941425.953308:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62d80 x1860112241145472/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941441 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941425.953313:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941425.953315:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941425.953316:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941425.953318:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941425.953318:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941425.953323:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1773941425.953324:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62d80 x1860112241145472/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941441 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941425.953326:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941425.953333:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241145600/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941425.953334:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241145472:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941425.953341:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941425.953344:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941425.953346:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941425.953348:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941425.953349:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941425.953351:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941425.953356:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1773941425.953357:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62d80 x1860112241145472/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941441 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941425.953359:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941425.953364:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941425.953370:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241145600:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941425.953372:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941425.953373:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:1.0:1773941425.953377:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6c00. 00000100:00000001:1.0:1773941425.953380:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:2.0:1773941425.953382:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941425.953383:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941425.953384:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941425.953389:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941425.953389:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000010:2.0:1773941425.953392:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce7800. 00010000:00000001:1.0:1773941425.953394:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:2.0:1773941425.953395:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941425.953397:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941425.953400:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000040:1.0:1773941425.953401:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=4 00000400:00000010:2.0:1773941425.953403:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9840. 00000020:00000001:1.0:1773941425.953405:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000400:00000010:2.0:1773941425.953407:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34ee0. 00010000:00000001:1.0:1773941425.953408:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000100:00000200:2.0:1773941425.953413:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241145600, portal 25 00010000:00010000:1.0:1773941425.953414:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c60 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941425.953416:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941425.953417:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:2.0:1773941425.953425:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241145600/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941435 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941425.953427:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941425.953432:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c60 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941425.953445:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941425.953445:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941425.953449:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c60 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941425.953452:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941425.953456:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941425.953458:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:2.0:1773941425.953459:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34770. 00010000:00010000:1.0:1773941425.953460:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c60 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941425.953464:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241145600, offset 0 00010000:00000001:1.0:1773941425.953466:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941425.953468:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941425.953471:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941425.953474:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941425.953474:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c60 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941425.953481:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f000 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:2.0:1773941425.953482:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:1.0:1773941425.953484:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941425.953487:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c60 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941425.953489:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941425.953494:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941425.953495:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941425.953497:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941425.953498:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000020:00000001:1.0:1773941425.953499:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941425.953502:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=5 00000400:00000200:2.0:1773941425.953504:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1773941425.953505:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000400:00000200:2.0:1773941425.953508:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941425.953509:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000400:00000200:2.0:1773941425.953512:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[10, 11] 00010000:00000001:1.0:1773941425.953517:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:2.0:1773941425.953518:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00010000:1.0:1773941425.953520:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c60 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941425.953523:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [117:117:256:4294967295] 192.168.204.143@tcp LPNI seq info [12:12:8:4294967295] 00000400:00000200:2.0:1773941425.953532:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:1.0:1773941425.953533:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941425.953537:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c60 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941425.953538:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941425.953547:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941425.953552:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0e00. 10000000:01000000:1.0:1773941425.953552:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941425.953557:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000800:00000200:2.0:1773941425.953558:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00000001:1.0:1773941425.953559:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941425.953562:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941425.953566:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941425.953570:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000800:00000200:2.0:1773941425.953571:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941425.953574:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000800:00000200:2.0:1773941425.953575:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0e00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941425.953577:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941425.953579:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941425.953581:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f000 with handle 0xa33a5be14eae1c60 from hash 00010000:00000001:1.0:1773941425.953583:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941425.953583:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941425.953584:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941425.953585:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:2.0:1773941425.953586:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941425.953587:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941425.953589:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941425.953592:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941425.953593:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941425.953593:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941425.953595:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:1.0:1773941425.953595:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941425.953597:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c60 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941425.953598:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294866316 00000100:00000001:2.0:1773941425.953605:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294866315 00010000:00000001:1.0:1773941425.953605:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941425.953607:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941425.953610:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294866316 00010000:00000001:1.0:1773941425.953610:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941425.953611:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:2.0:1773941425.953614:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294866315 00000100:00000001:2.0:1773941425.953617:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941425.953619:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941425.953620:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941425.953621:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941425.953623:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941425.953623:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941425.953626:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c60 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941425.953627:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:1.0:1773941425.953635:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec200 count: 0 00000100:00000001:2.0:1773941425.953636:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941425.953639:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941425.953641:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941425.953642:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941425.953643:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941425.953645:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941425.953649:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941425.953651:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941425.953655:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941425.953660:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62d80 x1860112241145472/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941441 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1773941425.953664:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1773941425.953668:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000800:00000010:3.0:1773941425.953670:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0e00. 02000000:00000001:1.0:1773941425.953672:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941425.953675:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735c00. 02000000:00000001:1.0:1773941425.953679:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941425.953680:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941425.953683:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000400:00000200:3.0:1773941425.953686:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:1.0:1773941425.953687:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941425.953691:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7400. 00000400:00000200:3.0:1773941425.953696:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 02000000:00000001:1.0:1773941425.953698:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941425.953700:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000400:00000200:3.0:1773941425.953701:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34770 00000100:00000010:1.0:1773941425.953736:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62d80. 00000100:00000001:1.0:1773941425.953740:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941425.953742:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941425.953744:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941425.953747:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941425.953749:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941425.953752:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0880. 00000020:00000010:1.0:1773941425.953756:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7400. 00000020:00000010:1.0:1773941425.953762:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b7800. 00000040:00000010:1.0:1773941425.953767:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7a00. 00000020:00000010:1.0:1773941425.953772:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000400:00000010:3.0:1773941425.953777:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34770. 00000020:00000010:1.0:1773941425.953777:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941425.953782:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000001:3.0:1773941425.953784:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941425.953789:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241145600/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941435 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00040000:00000010:1.0:1773941425.953796:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6e00. 00000100:00000001:3.0:1773941425.953802:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 20000000:00000010:1.0:1773941425.953804:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b7000. 00000100:00000040:3.0:1773941425.953808:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241145600/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941435 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000040:1.0:1773941425.953811:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941425.953816:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941425.953823:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941425.953823:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b7800. 00000100:00000001:3.0:1773941425.953829:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000040:00000010:1.0:1773941425.953829:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7a00. 00000020:00000010:1.0:1773941425.953832:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941425.953835:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941425.953838:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941425.953842:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6e00. 20000000:00000010:1.0:1773941425.953844:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941425.953846:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7400. 10000000:00000010:1.0:1773941425.953848:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0880. 10000000:01000000:1.0:1773941425.953850:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941425.953852:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941425.953853:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000800:00000001:3.0:1773941425.954982:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941425.954989:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941425.954991:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941425.954994:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941425.954998:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941425.955009:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941425.955019:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941425.955022:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941425.955101:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941425.955104:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321603700 00000400:00000200:3.0:1773941425.955108:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941425.955117:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xb0d [1] + 0 00000800:00000001:3.0:1773941425.955122:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941425.955127:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941425.955130:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941425.955133:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941425.955136:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941425.955140:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241145600/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941435 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941425.955151:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63800 x1860112241145600/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941435 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941425.955164:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941425.955177:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941425.955180:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941425.955194:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941425.955197:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941425.955200:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941425.955201:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941425.955203:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9840. 00000400:00000200:2.0:1773941425.955206:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34ee0 00000400:00000010:2.0:1773941425.955207:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34ee0. 00000100:00000001:2.0:1773941425.955209:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941425.955213:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241145600/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941435 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941425.955220:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241145600/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941435 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941425.955224:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941425.955225:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941425.955227:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941425.955233:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941425.955234:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941425.955235:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941425.955237:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941425.955239:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=4 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941425.955241:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941425.955243:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941425.955244:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941425.955245:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941425.955247:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941425.955248:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941425.955251:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241145600/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941435 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941425.955255:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941425.955257:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941425.955258:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941425.955260:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941425.955263:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941425.955264:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941425.955265:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941425.955266:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941425.955267:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941425.955268:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941425.955270:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941425.955272:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941425 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941425.955274:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941425.955278:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941425.955279:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941425.955280:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941425.955281:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941425.955282:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941425.955284:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941425.955285:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941425.955287:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941425.955292:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941425.955294:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941425.955296:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941425.955504:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941425.955508:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941425.955513:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241145600/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941435 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941425.955520:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241145600:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941425.955525:0:12152:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:0.0:1773941425.955526:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941425.955529:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941425.955532:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241145600/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941435 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1773941425.955535:0:12152:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:2.0:1773941425.955541:0:12152:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941425.955542:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941425.955543:0:12152:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941425.955545:0:12152:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941425.955547:0:12152:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941425.955550:0:12152:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:0.0:1773941425.955550:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:2.0:1773941425.955553:0:12152:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941425.955554:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce7800. 00000001:00000001:2.0:1773941425.955556:0:12152:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:0.0:1773941425.955558:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:2.0:1773941425.955559:0:12152:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:0.0:1773941425.955562:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:2.0:1773941425.955563:0:12152:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941425.955565:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 10000000:00000001:2.0:1773941425.955566:0:12152:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941425.955569:0:12152:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:0.0:1773941425.955569:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941425.955572:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734c00. 00000100:00080000:2.0:1773941425.955574:0:12152:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:0.0:1773941425.955575:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1773941425.955578:0:12152:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:0.0:1773941425.955579:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941425.955581:0:12152:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000010:0.0:1773941425.955582:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:2.0:1773941425.955584:0:12152:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941425.955586:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:2.0:1773941425.955588:0:12152:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941425.955588:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941425.955594:0:12152:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:0.0:1773941425.955594:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:2.0:1773941425.955597:0:12152:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941425.955600:0:12152:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:0.0:1773941425.955600:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941425.955603:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941425.955610:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1773941425.955611:0:12152:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:0.0:1773941425.955612:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941425.955613:0:12152:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941425.955616:0:12152:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941425.955617:0:12152:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941425.955620:0:12152:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941425.955622:0:12152:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941425.955623:0:12152:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941425.955625:0:12152:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941425.955626:0:12152:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941425.955940:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941425.955942:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941425.955944:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941425.955946:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:1.1:1773941425.959738:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec200. 00010000:00000010:1.1:1773941425.959748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f000. 00000020:00000001:0.0:1773941426.015853:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941426.015860:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941426.015865:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1773941427.039918:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941427.039924:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941427.039942:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1773941427.103853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1773941427.103864:0:37:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1773941427.103873:0:37:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941427.103878:0:37:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941427.103881:0:37:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941427.103884:0:37:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941427.103888:0:37:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941427.103894:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b98a69f00. 00000100:00000001:3.0:1773941427.103899:0:37:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941427.103909:0:37:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS req@ffff8b5b90275180 x1860112241145728/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941427.103917:0:37:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941427.103920:0:37:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941427.103924:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008177fcba] to pc [ptlrpcd_01_01+1] req@ffff8b5b90275180 x1860112241145728/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941427.103949:0:37:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941427.103953:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (202) 00000100:00000001:3.0:1773941427.103961:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941427.103974:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941427.103979:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.103979:0:2355:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941427.103981:0:2355:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941427.103987:0:2355:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241145728/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941427.103994:0:2355:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941427.103996:0:2355:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941427.103998:0:2355:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941427.104000:0:2355:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941427.104002:0:2355:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941427.104004:0:2355:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2355:1860112241145728:192.168.204.143@tcp:400:kworker.0 00000100:00000001:2.0:1773941427.104007:0:2355:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941427.104012:0:2355:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941427.104014:0:2355:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941427.104015:0:2355:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941427.104019:0:2355:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b90e7fe00. 02000000:00000001:2.0:1773941427.104020:0:2355:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941427.104026:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941427.104029:0:2355:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9840. 00000400:00000010:2.0:1773941427.104034:0:2355:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34ee0. 00000100:00000200:2.0:1773941427.104039:0:2355:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241145728, portal 25 00000100:00000001:2.0:1773941427.104041:0:2355:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941427.104042:0:2355:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:2.0:1773941427.104045:0:2355:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241145728/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941443 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941427.104051:0:2355:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941427.104053:0:2355:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941427.104054:0:2355:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f344c8. 00000100:00000200:2.0:1773941427.104056:0:2355:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241145728, offset 0 00000400:00000200:2.0:1773941427.104060:0:2355:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941427.104066:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941427.104072:0:2355:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941427.104081:0:2355:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941427.104084:0:2355:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941427.104087:0:2355:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941427.104089:0:2355:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941427.104091:0:2355:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[12, 11] 00000400:00000200:2.0:1773941427.104093:0:2355:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:2.0:1773941427.104096:0:2355:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [118:118:256:4294967295] 192.168.204.153@tcp LPNI seq info [13:13:8:4294967295] 00000400:00000200:2.0:1773941427.104098:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941427.104112:0:2355:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:2.0:1773941427.104119:0:2355:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:2.0:1773941427.104123:0:2355:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0800. 00000800:00000200:2.0:1773941427.104127:0:2355:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:2.0:1773941427.104132:0:2355:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:2.0:1773941427.104136:0:2355:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0800 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1773941427.104145:0:2355:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941427.104158:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941427.104160:0:2355:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294867467 00000100:00000001:2.0:1773941427.104166:0:2355:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294866316 00000100:00000001:2.0:1773941427.104169:0:2355:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294867467 00000100:00000001:2.0:1773941427.104172:0:2355:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294866316 00000100:00000001:2.0:1773941427.104177:0:2355:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941427.104179:0:2355:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.104181:0:2355:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.104183:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941427.104188:0:2355:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941427.104190:0:2355:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1773941427.104191:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941427.104192:0:2355:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941427.104193:0:2355:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.104195:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941427.104243:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941427.104246:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0800. 00000400:00000200:3.0:1773941427.104251:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941427.104255:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941427.104258:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f344c8 00000400:00000010:3.0:1773941427.104260:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f344c8. 00000100:00000001:3.0:1773941427.104263:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941427.104266:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241145728/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941443 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941427.104273:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941427.104277:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241145728/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941443 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941427.104283:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941427.104285:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941427.105069:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.105077:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941427.105079:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.105080:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941427.105084:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941427.105087:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941427.105121:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941427.105124:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941427.105127:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941427.105130:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321603780 00000400:00000200:3.0:1773941427.105133:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941427.105136:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xb1d [1] + 224 00000800:00000001:3.0:1773941427.105140:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.105145:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941427.105158:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941427.105161:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941427.105164:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941427.105166:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241145728/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941443 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941427.105181:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b90275180 x1860112241145728/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941443 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941427.105191:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941427.105197:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.105199:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941427.105213:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941427.105220:0:2355:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941427.105223:0:2355:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941427.105224:0:2355:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941427.105226:0:2355:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9840. 00000400:00000200:2.0:1773941427.105228:0:2355:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34ee0 00000400:00000010:2.0:1773941427.105230:0:2355:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34ee0. 00000100:00000001:2.0:1773941427.105231:0:2355:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941427.105235:0:2355:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241145728/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941443 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941427.105241:0:2355:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241145728/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941443 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941427.105246:0:2355:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941427.105247:0:2355:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941427.105248:0:2355:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941427.105250:0:2355:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941427.105251:0:2355:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941427.105253:0:2355:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941427.105254:0:2355:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941427.105259:0:2355:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=6 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941427.105262:0:2355:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941427.105264:0:2355:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941427.105265:0:2355:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941427.105267:0:2355:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941427.105269:0:2355:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941427.105272:0:2355:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941427.105275:0:2355:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941427.110182:0:2355:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941427.110185:0:2355:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941427.110186:0:2355:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941427.110188:0:2355:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941427.110194:0:2355:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241145728/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941443 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941427.110202:0:2355:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941427.110212:0:2355:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941427.110214:0:2355:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941427.110217:0:2355:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941427.110219:0:2355:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941427.110220:0:2355:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.110221:0:2355:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941427.110222:0:2355:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941427.110223:0:2355:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941427.110224:0:2355:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.110224:0:2355:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941427.110226:0:2355:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941427.110228:0:2355:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941427.110231:0:2355:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941427.110233:0:2355:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 196 00000400:00000200:2.0:1773941427.110237:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941427.110242:0:2355:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941427.110244:0:2355:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941427.110246:0:2355:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941427.110248:0:2355:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941427.110250:0:2355:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:2.0:1773941427.110252:0:2355:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941427.110255:0:2355:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941427.110257:0:2355:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941427.110259:0:2355:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941427.110260:0:2355:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941427.110262:0:2355:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:2.0:1773941427.110263:0:2355:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941427.110264:0:2355:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941427.110265:0:2355:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941427.110266:0:2355:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:2.0:1773941427.110268:0:2355:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:2.0:1773941427.110270:0:2355:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:2.0:1773941427.110271:0:2355:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941427.110276:0:2355:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941427.110279:0:2355:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941427.110280:0:2355:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.110282:0:2355:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941427.110283:0:2355:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941427.110288:0:2355:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021c380. 00000020:00000040:2.0:1773941427.110290:0:2355:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:2.0:1773941427.110293:0:2355:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941427.110294:0:2355:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.110296:0:2355:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941427.110298:0:2355:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941427.110301:0:2355:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941427.110306:0:2355:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0ce5c00. 00000100:00000001:2.0:1773941427.110310:0:2355:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941427.110314:0:2355:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b9021c380 x1860112241145856/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941427.110320:0:2355:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941427.110323:0:2355:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941427.110327:0:2355:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae998bae] to pc [ptlrpcd_rcv+-1] req@ffff8b5b9021c380 x1860112241145856/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941427.110349:0:2355:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.110353:0:2355:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941427.110354:0:2355:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:2.0:1773941427.110360:0:2355:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241145728/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941443 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941427.110368:0:2355:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941427.110371:0:2355:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941427.110373:0:2355:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941427.110377:0:2355:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241145728/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941443 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941427.110383:0:2355:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2355:1860112241145728:192.168.204.143@tcp:400:kworker.0 00000100:00000001:2.0:1773941427.110392:0:2355:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941427.110392:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941427.110396:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941427.110397:0:2355:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941427.110399:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941427.110402:0:2355:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241145728/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941443 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941427.110406:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021c380 x1860112241145856/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941427.110410:0:2355:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941427.110413:0:2355:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941427.110416:0:2355:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b90e7fe00. 00000100:00000001:0.0:1773941427.110418:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1773941427.110420:0:2355:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1773941427.110421:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941427.110424:0:2355:0:(genops.c:1236:class_import_put()) Process entered 02000000:00000001:0.0:1773941427.110425:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:2.0:1773941427.110426:0:2355:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 02000000:00000001:0.0:1773941427.110428:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1773941427.110430:0:2355:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:0.0:1773941427.110431:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941427.110432:0:2355:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b98a69f00. 00000100:00100000:0.0:1773941427.110436:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241145856:192.168.204.143@tcp:250:ptlrpcd_01_01.0 02000000:00000001:2.0:1773941427.110437:0:2355:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941427.110440:0:2355:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941427.110441:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:2.0:1773941427.110442:0:2355:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 02000000:00000001:0.0:1773941427.110444:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1773941427.110445:0:2355:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941427.110447:0:2355:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1773941427.110447:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.110450:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1773941427.110450:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941427.110455:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0c00. 00000100:00000001:2.0:1773941427.110457:0:2355:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941427.110458:0:2355:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.110460:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941427.110461:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941427.110464:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941427.110469:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941427.110472:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941427.110476:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941427.110480:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241145856, portal 25 00000100:00000001:0.0:1773941427.110482:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941427.110483:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109257600 : -128249600294016 : ffff8b5b9021c380) 00000100:00000040:0.0:1773941427.110487:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021c380 x1860112241145856/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941427.110492:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941427.110494:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941427.110496:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c088. 00000100:00000200:0.0:1773941427.110498:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241145856, offset 0 00000400:00000200:0.0:1773941427.110502:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941427.110514:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941427.110518:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941427.110522:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941427.110525:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941427.110527:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941427.110529:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941427.110531:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[12, 13] 00000400:00000200:0.0:1773941427.110533:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941427.110536:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [119:119:256:4294967295] 192.168.204.143@tcp LPNI seq info [14:14:8:4294967295] 00000400:00000200:0.0:1773941427.110538:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941427.110545:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941427.110549:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941427.110552:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00000800:00000200:0.0:1773941427.110557:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941427.110563:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941427.110565:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:0.0:1773941427.110575:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941427.110582:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941427.110584:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294867473 00000100:00000001:0.0:1773941427.110589:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294867467 00000100:00000001:0.0:1773941427.110593:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294867473 00000100:00000001:0.0:1773941427.110597:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294867467 00000100:00000001:0.0:1773941427.110600:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941427.110602:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941427.110605:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941427.110606:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941427.110610:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941427.110611:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1773941427.110612:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941427.110613:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941427.110615:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941427.110616:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941427.110662:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941427.110667:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000400:00000200:3.0:1773941427.110672:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941427.110677:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941427.110681:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c088 00000400:00000010:3.0:1773941427.110684:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c088. 00000100:00000001:3.0:1773941427.110687:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941427.110692:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021c380 x1860112241145856/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941427.110701:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941427.110728:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021c380 x1860112241145856/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941437 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941427.110736:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941427.110739:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941427.115673:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.115685:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941427.115688:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.115690:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941427.115695:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941427.115700:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941427.115723:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941427.115726:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941427.115730:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941427.115732:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321603800 00000400:00000200:3.0:1773941427.115735:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941427.115739:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xb2d [1] + 0 00000800:00000001:3.0:1773941427.115754:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.115760:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941427.115762:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941427.115764:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941427.115769:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941427.115774:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021c380 x1860112241145856/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941437 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941427.115782:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b9021c380 x1860112241145856/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941437 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941427.115794:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941427.115799:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.115801:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941427.115831:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941427.115836:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941427.115842:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941427.115844:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941427.115847:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941427.115851:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941427.115853:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941427.115856:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941427.115862:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021c380 x1860112241145856/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941437 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941427.115872:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021c380 x1860112241145856/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941437 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941427.115880:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941427.115882:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941427.115884:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941427.115887:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941427.115889:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941427.115892:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941427.115894:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941427.115899:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=6 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941427.115910:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941427.115912:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941427.115914:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941427.115916:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941427.115919:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941427.115921:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941427.115926:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021c380 x1860112241145856/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941437 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941427.115933:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941427.115936:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941427.115938:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941427.115940:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941427.115946:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941427.115949:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941427.115950:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941427.115953:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941427.115955:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941427.115958:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941427.115960:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941427.115963:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941427 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941427.115966:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941427.115971:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941427.115974:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941427.115977:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941427.115979:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941427.115981:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941427.115984:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941427.115987:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941427.115989:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941427.115993:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941427.115995:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941427.115998:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941427.116223:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.116227:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941427.116232:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021c380 x1860112241145856/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941437 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941427.116240:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241145856:192.168.204.143@tcp:250:ptlrpcd_01_01.0 00000100:00000001:2.0:1773941427.116244:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941427.116246:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941427.116247:0:12153:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941427.116249:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021c380 x1860112241145856/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941437 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941427.116256:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00080000:0.0:1773941427.116261:0:12153:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 02000000:00000001:2.0:1773941427.116264:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941427.116267:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0c00. 00000100:00100000:0.0:1773941427.116267:0:12153:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:2.0:1773941427.116273:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1773941427.116273:0:12153:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000001:2.0:1773941427.116277:0:2351:0:(genops.c:1236:class_import_put()) Process entered 10000000:01000000:0.0:1773941427.116277:0:12153:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000040:2.0:1773941427.116279:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941427.116281:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:0.0:1773941427.116282:0:12153:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941427.116283:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0ce5c00. 00000001:00000001:0.0:1773941427.116286:0:12153:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941427.116287:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941427.116289:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941427.116290:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021c380. 00000100:00000001:2.0:1773941427.116293:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:0.0:1773941427.116293:0:12153:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.116295:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941427.116298:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:0.0:1773941427.116298:0:12153:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941427.116300:0:12153:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941427.116303:0:12153:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941427.116305:0:12153:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.116306:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941427.116307:0:12153:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941427.116309:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941427.116313:0:12153:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941427.116314:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941427.116317:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941427.116318:0:12153:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941427.116322:0:12153:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941427.116324:0:12153:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941427.116326:0:12153:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941427.116329:0:12153:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941427.116330:0:12153:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941427.116332:0:12153:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941427.116350:0:12153:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941427.116352:0:12153:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941427.116355:0:12153:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941427.116358:0:12153:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941427.120471:0:12153:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941427.120476:0:12153:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941427.120478:0:12153:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941427.120481:0:12153:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941427.120482:0:12153:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941427.679811:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941427.679818:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941427.679823:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941427.679825:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941427.679827:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941427.679828:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941427.679831:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941427.679833:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941427.679835:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941427.679838:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941427.679839:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941427.679849:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62d80. 00000020:00000040:1.0:1773941427.679853:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941427.679856:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941427.679859:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941427.679874:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941427.679880:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7400. 00000100:00000001:1.0:1773941427.679886:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941427.679889:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941427.679891:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941427.679897:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec200. 00010000:00000001:1.0:1773941427.679901:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941427.679903:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f000. 00000020:00000001:1.0:1773941427.679906:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941427.679908:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f000 with handle 0xa33a5be14eae1c67 to hash 00000020:00000001:1.0:1773941427.679909:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941427.679910:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941427.679912:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941427.679915:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941427.679917:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941427.679920:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c67 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941427.679928:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c67 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941427.679939:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c67 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941427.679945:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941427.679947:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941427.679953:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6c00. 00000100:00000001:1.0:1773941427.679956:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258984960 : -128249450566656 : ffff8b5b990e6c00) 00000100:00000001:1.0:1773941427.679958:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941427.679960:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121081216 : -128249588470400 : ffff8b5b90d62d80) 00000100:00000001:1.0:1773941427.679964:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941427.679967:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941427.679969:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941427.679970:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941427.679982:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62d80 x1860112241145984/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941427.679989:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941427.679990:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941427.679993:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941427.679995:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941427.679996:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941427.679998:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241145984:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941427.680001:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941427.680006:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941427.680007:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941427.680009:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941427.680012:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735c00. 02000000:00000001:1.0:1773941427.680013:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941427.680019:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941427.680021:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941427.680027:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941427.680032:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241145984, portal 25 00000100:00000001:1.0:1773941427.680033:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941427.680034:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121081216 : -128249588470400 : ffff8b5b90d62d80) 00000100:00000040:1.0:1773941427.680038:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62d80 x1860112241145984/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941443 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941427.680043:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941427.680045:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941427.680046:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1e58. 00000100:00000200:1.0:1773941427.680048:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241145984, offset 0 00000400:00000200:1.0:1773941427.680052:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941427.680058:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941427.680063:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941427.680067:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941427.680070:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941427.680073:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941427.680078:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941427.680080:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[14, 13] 00000400:00000200:1.0:1773941427.680083:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941427.680086:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [120:120:256:4294967295] 192.168.204.153@tcp LPNI seq info [15:15:8:4294967295] 00000400:00000200:1.0:1773941427.680089:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941427.680096:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941427.680099:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941427.680103:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024bb00. 00000800:00000200:1.0:1773941427.680108:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941427.680114:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941427.680117:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024bb00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941427.680130:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941427.680132:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941427.680134:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294868043 00000100:00000001:1.0:1773941427.680152:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294867473 00000100:00000001:1.0:1773941427.680155:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294868043 00000100:00000001:1.0:1773941427.680158:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294867473 00000100:00000001:1.0:1773941427.680160:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941427.680162:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941427.680164:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941427.680165:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941427.680168:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6c00 going to sleep for 11 seconds 00000100:00000001:1.0:1773941427.680170:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941427.680172:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941427.680243:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941427.680247:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024bb00. 00000400:00000200:3.0:1773941427.680252:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941427.680257:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941427.680260:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1e58 00000400:00000010:3.0:1773941427.680262:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1e58. 00000100:00000001:3.0:1773941427.680265:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941427.680268:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62d80 x1860112241145984/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941443 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941427.680275:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941427.680277:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d62d80 x1860112241145984/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941443 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941427.680282:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941427.680283:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941427.681217:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.681225:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941427.681227:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.681228:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941427.681232:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941427.681235:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941427.681239:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941427.681275:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941427.681278:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941427.681281:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321603880 00000400:00000200:3.0:1773941427.681284:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941427.681287:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xb35 [1] + 224 00000800:00000001:3.0:1773941427.681291:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.681296:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941427.681298:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941427.681300:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941427.681303:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941427.681306:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62d80 x1860112241145984/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941443 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941427.681312:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d62d80 x1860112241145984/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941443 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941427.681325:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941427.681330:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.681332:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941427.681362:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941427.681367:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941427.681368:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941427.681370:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941427.681373:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941427.681375:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941427.681376:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941427.681386:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62d80 x1860112241145984/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941443 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941427.681392:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62d80 x1860112241145984/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941443 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941427.681397:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941427.681398:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941427.681400:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941427.681402:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941427.681403:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941427.681405:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941427.681406:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941427.681411:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=6 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941427.681413:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941427.681415:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941427.681418:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941427.684641:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941427.684651:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941427.684658:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941427.684664:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941427.684669:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941427.684672:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941427.684674:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941427.684676:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941427.684677:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941427.684683:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d62d80 x1860112241145984/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941443 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941427.684691:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941427.684692:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941427.684693:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941427.684696:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941427.684699:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941427.684701:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941427.684746:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941427.684747:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941427.684749:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941427.684750:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941427.684751:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941427.684753:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941427.684754:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941427.684757:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941427.684760:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 197 00000400:00000200:1.0:1773941427.684767:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941427.684773:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941427.684777:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941427.684780:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941427.684782:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941427.684784:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941427.684787:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941427.684789:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941427.684791:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941427.684794:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941427.684796:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941427.684798:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941427.684800:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941427.684804:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941427.684805:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941427.684806:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941427.684808:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941427.684811:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941427.684812:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941427.684815:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941427.684819:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941427.684821:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941427.684823:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941427.684829:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941427.684834:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62680. 00000020:00000040:1.0:1773941427.684837:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941427.684841:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941427.684842:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941427.684845:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941427.684847:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941427.684850:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941427.684857:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734c00. 00000100:00000001:1.0:1773941427.684863:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941427.684870:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d62680 x1860112241146112/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941427.684880:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941427.684884:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941427.684890:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048151604] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d62680 x1860112241146112/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941427.684911:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941427.684914:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941427.684915:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941427.684919:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941427.684922:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941427.684925:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941427.684928:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6c00 going to sleep for 1 seconds 00000100:00000001:2.0:1773941427.684931:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941427.684932:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941427.684937:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941427.684938:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941427.684940:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941427.684957:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62680 x1860112241146112/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941427.684957:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62d80 x1860112241145984/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941443 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941427.685025:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941427.685025:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941427.685032:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941427.685033:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941427.685046:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941427.685046:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1773941427.685057:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941427.685058:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941427.685070:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941427.685075:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62d80 x1860112241145984/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941443 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941427.685076:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241146112:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941427.685098:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941427.685103:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941427.685106:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941427.685111:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941427.685116:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce5c00. 02000000:00000001:2.0:1773941427.685122:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941427.685131:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941427.685134:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9840. 00000100:00100000:1.0:1773941427.685134:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241145984:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941427.685155:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1773941427.685157:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34ee0. 00000100:00000001:1.0:1773941427.685162:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000200:2.0:1773941427.685164:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241146112, portal 25 00000100:00000001:2.0:1773941427.685171:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941427.685172:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941427.685173:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121079424 : -128249588472192 : ffff8b5b90d62680) 00000100:00000001:1.0:1773941427.685175:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941427.685186:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62680 x1860112241146112/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941427.685186:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62d80 x1860112241145984/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941443 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941427.685252:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1773941427.685252:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941427.685263:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000100:00000010:1.0:1773941427.685264:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6c00. 00000400:00000010:2.0:1773941427.685265:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34880. 00000100:00000200:2.0:1773941427.685272:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241146112, offset 0 00000100:00000001:1.0:1773941427.685275:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000400:00000200:2.0:1773941427.685279:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000100:00000001:1.0:1773941427.685280:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941427.685287:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941427.685288:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000200:2.0:1773941427.685295:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:1.0:1773941427.685296:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941427.685303:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941427.685304:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:1.0:1773941427.685311:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=4 00000400:00000200:2.0:1773941427.685312:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000020:00000001:1.0:1773941427.685320:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000400:00000200:2.0:1773941427.685321:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941427.685330:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941427.685331:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000400:00000200:2.0:1773941427.685342:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[14, 15] 00010000:00010000:1.0:1773941427.685343:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c67 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941427.685354:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941427.685364:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [121:121:256:4294967295] 192.168.204.143@tcp LPNI seq info [16:16:8:4294967295] 00010000:00000001:1.0:1773941427.685365:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:2.0:1773941427.685378:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941427.685378:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c67 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941427.685394:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:2.0:1773941427.685398:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00010000:1.0:1773941427.685400:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c67 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941427.685407:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941427.685415:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0600. 00000800:00000200:2.0:1773941427.685423:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941427.685423:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941427.685432:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941427.685435:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:2.0:1773941427.685436:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0600 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:1.0:1773941427.685443:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c67 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941427.685451:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941427.685456:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941427.685457:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941427.685462:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294868048 00010000:00000001:1.0:1773941427.685463:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941427.685471:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000001:2.0:1773941427.685472:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294868043 00010000:00010000:1.0:1773941427.685478:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c67 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941427.685487:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294868048 00000100:00000001:2.0:1773941427.685493:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294868043 00010000:00010000:1.0:1773941427.685498:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f000 already unused, calling callback (ffffffffc1482c20) 00000100:00000001:2.0:1773941427.685504:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:00000001:1.0:1773941427.685505:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:2.0:1773941427.685511:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 10000000:00010000:1.0:1773941427.685513:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c67 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941427.685514:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.685525:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1773941427.685535:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941427.685537:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:2.0:1773941427.685539:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941427.685540:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:2.0:1773941427.685546:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000001:1.0:1773941427.685546:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000010:3.0:1773941427.685547:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0600. 00000020:00000040:1.0:1773941427.685550:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=5 00000100:00000001:2.0:1773941427.685555:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1773941427.685559:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000100:00000001:2.0:1773941427.685560:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1773941427.685561:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:1.0:1773941427.685569:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000100:00000001:2.0:1773941427.685570:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941427.685578:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941427.685579:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941427.685581:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941427.685591:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c67 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:1.0:1773941427.685598:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941427.685601:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c67 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941427.685605:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941427.685606:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941427.685607:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941427.685610:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941427.685613:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941427.685623:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941427.685625:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:3.0:1773941427.685627:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34880 00010000:00000001:1.0:1773941427.685629:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000010:3.0:1773941427.685631:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34880. 00010000:00000001:1.0:1773941427.685634:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941427.685636:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:1.0:1773941427.685643:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f000 with handle 0xa33a5be14eae1c67 from hash 00000100:00000200:3.0:1773941427.685644:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62680 x1860112241146112/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941437 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941427.685652:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941427.685659:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941427.685660:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000001:1.0:1773941427.685665:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000040:3.0:1773941427.685670:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62680 x1860112241146112/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941437 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941427.685671:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941427.685677:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941427.685682:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941427.685690:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941427.685692:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941427.685693:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941427.685699:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1773941427.685700:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941427.685732:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941427.685737:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c67 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941427.685743:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941427.685745:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941427.685746:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941427.685748:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941427.685749:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941427.685750:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941427.685751:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941427.685752:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c67 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941427.685758:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec200 count: 0 00010000:00000001:1.0:1773941427.685771:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941427.685772:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941427.685775:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941427.685779:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62d80 x1860112241145984/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941443 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941427.685784:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941427.685786:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941427.685793:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735c00. 02000000:00000001:1.0:1773941427.685796:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941427.685797:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941427.685799:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941427.685800:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941427.685802:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7400. 02000000:00000001:1.0:1773941427.685805:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941427.685806:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941427.685808:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62d80. 00000100:00000001:1.0:1773941427.685811:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941427.685812:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941427.685813:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941427.685815:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941427.685817:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941427.685820:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0880. 00000020:00000010:1.0:1773941427.685823:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7400. 00000020:00000010:1.0:1773941427.685827:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b7000. 00000040:00000010:1.0:1773941427.685830:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941427.685834:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941427.685838:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941427.685842:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941427.685846:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7a00. 20000000:00000010:1.0:1773941427.685849:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b7800. 10000000:00000040:1.0:1773941427.685852:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941427.685854:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941427.685859:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b7000. 00000040:00000010:1.0:1773941427.685862:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941427.685865:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941427.685870:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941427.685873:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941427.685878:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7a00. 20000000:00000010:1.0:1773941427.685881:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941427.685883:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7400. 10000000:00000010:1.0:1773941427.685886:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0880. 10000000:01000000:1.0:1773941427.685889:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941427.685896:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941427.685899:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941427.685902:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941427.685904:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941427.685906:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941427.685907:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941427.686773:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.686788:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941427.686791:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.686795:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941427.686802:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941427.686810:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941427.686818:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941427.686822:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941427.686827:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941427.686831:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321603900 00000400:00000200:3.0:1773941427.686835:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941427.686841:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xb3d [1] + 0 00000800:00000001:3.0:1773941427.686848:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.686857:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941427.686860:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941427.686864:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941427.686870:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941427.686876:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62680 x1860112241146112/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941437 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941427.686886:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d62680 x1860112241146112/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941437 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941427.686901:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941427.686910:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941427.686915:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941427.686925:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941427.686928:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941427.686934:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941427.686935:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941427.686937:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9840. 00000400:00000200:2.0:1773941427.686941:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34ee0 00000400:00000010:2.0:1773941427.686942:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34ee0. 00000100:00000001:2.0:1773941427.686944:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941427.686949:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62680 x1860112241146112/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941437 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941427.686956:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62680 x1860112241146112/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941437 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941427.686961:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941427.686962:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941427.686964:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941427.686965:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941427.686967:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941427.686968:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941427.686970:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941427.686974:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=6 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941427.686976:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941427.686978:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941427.686979:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941427.686986:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941427.686987:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.686989:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941427.686992:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62680 x1860112241146112/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941437 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941427.686997:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941427.686999:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941427.687001:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941427.687002:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941427.687007:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941427.687009:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941427.687010:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941427.687012:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941427.687014:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941427.687016:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941427.687018:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941427.687020:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941427 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941427.687022:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941427.687026:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941427.687028:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941427.687030:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941427.687031:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.687032:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941427.687035:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941427.687036:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941427.687038:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941427.687041:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941427.687043:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941427.687044:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941427.687288:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.687290:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941427.687294:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62680 x1860112241146112/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941437 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941427.687306:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241146112:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941427.687308:0:12154:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941427.687310:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941427.687313:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941427.687316:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62680 x1860112241146112/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941437 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941427.687323:0:12154:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:2.0:1773941427.687327:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941427.687332:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:1.0:1773941427.687333:0:12154:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000010:2.0:1773941427.687335:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce5c00. 00000100:00000001:1.0:1773941427.687338:0:12154:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941427.687339:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941427.687342:0:2351:0:(genops.c:1236:class_import_put()) Process entered 10000000:01000000:1.0:1773941427.687342:0:12154:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000040:2.0:1773941427.687347:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941427.687350:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941427.687350:0:12154:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941427.687359:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734c00. 00000001:00000001:1.0:1773941427.687360:0:12154:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941427.687365:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:1.0:1773941427.687367:0:12154:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941427.687368:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941427.687375:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62680. 00000001:00000001:1.0:1773941427.687376:0:12154:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941427.687381:0:12154:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941427.687382:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941427.687388:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941427.687390:0:12154:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.687393:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941427.687395:0:12154:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941427.687400:0:12154:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941427.687401:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1773941427.687405:0:12154:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941427.687406:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941427.687414:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941427.687414:0:12154:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941427.687420:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941427.687420:0:12154:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941427.687425:0:12154:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941427.687430:0:12154:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941427.687433:0:12154:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941427.687434:0:12154:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941427.687437:0:12154:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941427.687453:0:12154:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941427.687454:0:12154:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941427.687456:0:12154:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941427.687460:0:12154:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941427.687463:0:12154:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941427.687466:0:12154:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941427.687467:0:12154:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941427.687469:0:12154:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941427.687471:0:12154:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941427.691743:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec200. 00010000:00000010:1.1:1773941427.691753:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f000. 00000020:00000001:0.0:1773941428.063880:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941428.063885:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941428.063888:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:2.0:1773941428.895826:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941428.895834:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:2.0:1773941428.895838:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941428.895841:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941428.895842:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941428.895843:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941428.895846:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941428.895848:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941428.895850:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:2.0:1773941428.895852:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941428.895854:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941428.895884:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021c380. 00000020:00000040:2.0:1773941428.895889:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941428.895894:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941428.895897:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941428.895903:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941428.895909:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b90e7fc00. 00000100:00000001:2.0:1773941428.895916:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941428.895920:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941428.895922:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941428.895930:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5ba0d64d00. 00010000:00000001:2.0:1773941428.895936:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941428.895938:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5bc2151e00. 00000020:00000001:2.0:1773941428.895942:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941428.895945:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5bc2151e00 with handle 0xa33a5be14eae1c6e to hash 00000020:00000001:2.0:1773941428.895946:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941428.895947:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615824947289600 : -128248762262016 : ffff8b5bc2151e00) 00010000:00000001:2.0:1773941428.895950:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615824947289600 : -128248762262016 : ffff8b5bc2151e00) 00010000:00000001:2.0:1773941428.895953:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941428.895956:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941428.895959:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c6e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941428.895968:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c6e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941428.895978:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c6e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941428.895982:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941428.895983:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941428.895989:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b54480. 00000100:00000001:2.0:1773941428.895990:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387359872 : -128249322191744 : ffff8b5ba0b54480) 00000100:00000001:2.0:1773941428.895993:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941428.895994:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109257600 : -128249600294016 : ffff8b5b9021c380) 00000100:00000001:2.0:1773941428.896002:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941428.896006:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941428.896008:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941428.896010:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941428.896024:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021c380 x1860112241146240/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941428.896031:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941428.896033:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941428.896036:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941428.896039:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941428.896040:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941428.896042:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021c380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241146240:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941428.896046:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941428.896052:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941428.896054:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941428.896057:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941428.896061:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce6400. 02000000:00000001:2.0:1773941428.896062:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941428.896070:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941428.896073:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9840. 00000400:00000010:2.0:1773941428.896080:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34ee0. 00000100:00000200:2.0:1773941428.896087:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241146240, portal 25 00000100:00000001:2.0:1773941428.896089:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941428.896090:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109257600 : -128249600294016 : ffff8b5b9021c380) 00000100:00000040:2.0:1773941428.896093:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021c380 x1860112241146240/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941444 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941428.896099:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941428.896100:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941428.896102:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34990. 00000100:00000200:2.0:1773941428.896104:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241146240, offset 0 00000400:00000200:2.0:1773941428.896109:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941428.896137:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941428.896144:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941428.896153:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941428.896157:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941428.896160:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941428.896162:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941428.896165:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[16, 15] 00000400:00000200:2.0:1773941428.896167:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:2.0:1773941428.896170:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [122:122:256:4294967295] 192.168.204.153@tcp LPNI seq info [17:17:8:4294967295] 00000400:00000200:2.0:1773941428.896174:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941428.896183:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:2.0:1773941428.896187:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:2.0:1773941428.896191:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0500. 00000800:00000200:2.0:1773941428.896197:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:2.0:1773941428.896204:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:2.0:1773941428.896208:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0500 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941428.896226:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941428.896229:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941428.896231:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294869259 00000100:00000001:2.0:1773941428.896237:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294868048 00000100:00000001:2.0:1773941428.896242:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294869259 00000100:00000001:2.0:1773941428.896244:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294868048 00000100:00000001:2.0:1773941428.896246:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941428.896248:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941428.896251:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941428.896252:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941428.896255:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b54480 going to sleep for 11 seconds 00000100:00000001:2.0:1773941428.896258:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941428.896261:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941428.896381:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941428.896387:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0500. 00000400:00000200:3.0:1773941428.896394:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941428.896400:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941428.896406:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34990 00000400:00000010:3.0:1773941428.896409:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34990. 00000100:00000001:3.0:1773941428.896413:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941428.896419:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021c380 x1860112241146240/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941444 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941428.896441:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941428.896445:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b9021c380 x1860112241146240/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941444 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941428.896452:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941428.896454:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941428.897567:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941428.897578:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941428.897580:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941428.897582:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941428.897586:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941428.897590:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941428.897595:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941428.897597:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941428.897601:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941428.897604:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321603980 00000400:00000200:3.0:1773941428.897607:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941428.897644:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xb4d [1] + 224 00000800:00000001:3.0:1773941428.897652:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941428.897668:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941428.897671:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941428.897674:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941428.897679:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941428.897684:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021c380 x1860112241146240/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941444 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941428.897692:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b9021c380 x1860112241146240/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941444 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941428.897735:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941428.897743:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941428.897745:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941428.897754:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941428.897761:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941428.897763:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941428.897765:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9840. 00000400:00000200:1.0:1773941428.897768:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34ee0 00000400:00000010:1.0:1773941428.897770:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34ee0. 00000100:00000001:1.0:1773941428.897773:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941428.897776:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021c380 x1860112241146240/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941444 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941428.897784:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021c380 x1860112241146240/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941444 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941428.897790:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941428.897791:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941428.897793:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941428.897796:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941428.897798:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941428.897799:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941428.897802:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941428.897807:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=7 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941428.897810:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941428.897812:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941428.897815:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941428.897817:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941428.897820:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941428.897826:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941428.897829:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941428.897833:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941428.897835:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941428.897837:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941428.897839:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941428.897840:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941428.897842:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b9021c380 x1860112241146240/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941444 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941428.897853:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941428.897854:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941428.897855:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941428.897858:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941428.897860:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941428.897861:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941428.897862:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941428.897863:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941428.897864:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941428.897865:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941428.897866:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941428.897868:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941428.897869:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941428.897871:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941428.897873:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 198 00000400:00000200:1.0:1773941428.897876:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941428.897879:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941428.897882:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941428.897884:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941428.897886:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941428.897887:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941428.897890:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941428.897891:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941428.897892:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941428.897894:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941428.897896:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941428.897897:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941428.897899:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941428.897900:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941428.897901:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941428.897902:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941428.897904:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941428.897911:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941428.897913:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941428.897915:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941428.897918:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941428.897919:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941428.897921:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941428.897923:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941428.897926:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62d80. 00000020:00000040:1.0:1773941428.897928:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941428.897931:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941428.897932:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941428.897934:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941428.897936:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941428.897938:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941428.897941:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0735c00. 00000100:00000001:1.0:1773941428.897944:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941428.897948:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d62d80 x1860112241146368/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941428.897956:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941428.897959:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941428.897963:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002abf64a5] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d62d80 x1860112241146368/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941428.897983:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941428.897985:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941428.897988:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941428.897991:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941428.897993:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941428.897995:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941428.897997:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b54480 going to sleep for 1 seconds 00000100:00000001:1.0:1773941428.898000:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941428.898002:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941428.898008:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021c380 x1860112241146240/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941444 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941428.898011:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941428.898016:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941428.898019:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941428.898029:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62d80 x1860112241146368/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941428.898029:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941428.898032:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941428.898034:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941428.898035:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941428.898045:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021c380 x1860112241146240/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941444 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941428.898057:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941428.898061:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941428.898061:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021c380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241146240:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941428.898068:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941428.898070:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941428.898072:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941428.898073:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941428.898079:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941428.898080:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941428.898082:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1773941428.898089:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241146368:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000040:1.0:1773941428.898089:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021c380 x1860112241146240/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941444 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941428.898096:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941428.898101:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941428.898102:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941428.898104:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941428.898105:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b54480. 02000000:00000001:2.0:1773941428.898109:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941428.898110:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941428.898124:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000010:2.0:1773941428.898128:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce6800. 00010000:00000001:1.0:1773941428.898129:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:2.0:1773941428.898133:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941428.898133:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941428.898136:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941428.898138:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5bc2151e00 refcount=4 00000400:00000200:2.0:1773941428.898142:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:1.0:1773941428.898143:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824947289600 : -128248762262016 : ffff8b5bc2151e00) 00010000:00000001:1.0:1773941428.898147:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615824947289600 : -128248762262016 : ffff8b5bc2151e00) 00000400:00000010:2.0:1773941428.898149:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af98f0. 00010000:00010000:1.0:1773941428.898151:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c6e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941428.898156:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34d48. 00010000:00000001:1.0:1773941428.898158:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000200:2.0:1773941428.898162:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241146368, portal 25 00010000:00010000:1.0:1773941428.898162:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c6e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941428.898166:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941428.898168:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121081216 : -128249588470400 : ffff8b5b90d62d80) 00010000:00000001:1.0:1773941428.898177:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000040:2.0:1773941428.898178:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62d80 x1860112241146368/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941438 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941428.898180:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c6e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941428.898197:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941428.898200:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941428.898201:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:1.0:1773941428.898203:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c6e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941428.898208:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941428.898213:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000010:2.0:1773941428.898215:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34bb0. 00010000:00000001:1.0:1773941428.898215:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941428.898221:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241146368, offset 0 00010000:00000001:1.0:1773941428.898221:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941428.898224:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c6e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941428.898230:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941428.898233:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5bc2151e00 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941428.898235:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941428.898238:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00010000:1.0:1773941428.898239:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c6e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941428.898245:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:1.0:1773941428.898248:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:2.0:1773941428.898250:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941428.898252:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941428.898256:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000020:00000001:1.0:1773941428.898256:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941428.898260:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5bc2151e00 refcount=5 00000400:00000200:2.0:1773941428.898261:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1773941428.898265:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824947289600 : -128248762262016 : ffff8b5bc2151e00) 00000400:00000200:2.0:1773941428.898267:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941428.898270:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615824947289600 : -128248762262016 : ffff8b5bc2151e00) 00010000:00000001:1.0:1773941428.898274:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:2.0:1773941428.898279:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[16, 17] 00010000:00010000:1.0:1773941428.898280:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c6e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941428.898290:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 10000000:00000001:1.0:1773941428.898294:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941428.898297:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [123:123:256:4294967295] 192.168.204.143@tcp LPNI seq info [18:18:8:4294967295] 00000400:00000200:2.0:1773941428.898303:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00010000:1.0:1773941428.898305:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c6e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941428.898317:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 10000000:01000000:1.0:1773941428.898319:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941428.898325:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000800:00000200:2.0:1773941428.898326:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 10000000:00000001:1.0:1773941428.898328:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1773941428.898332:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0300. 00010000:00010000:1.0:1773941428.898333:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941428.898338:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000800:00000200:2.0:1773941428.898342:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941428.898343:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941428.898346:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941428.898347:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000800:00000200:2.0:1773941428.898350:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941428.898350:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941428.898354:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5bc2151e00 with handle 0xa33a5be14eae1c6e from hash 00000800:00000200:2.0:1773941428.898355:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0300 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941428.898358:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941428.898360:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941428.898361:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941428.898361:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941428.898363:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941428.898364:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941428.898366:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941428.898367:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941428.898367:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941428.898370:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941428.898372:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941428.898373:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941428.898375:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c6e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941428.898376:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294869261 00000100:00000001:2.0:1773941428.898382:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294869259 00010000:00000001:1.0:1773941428.898383:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941428.898385:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941428.898387:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:2.0:1773941428.898389:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294869261 00010000:00000001:1.0:1773941428.898389:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941428.898393:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941428.898394:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294869259 00010000:00000001:1.0:1773941428.898396:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941428.898398:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941428.898399:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941428.898401:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941428.898404:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151e00/0xa33a5be14eae1c6e lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941428.898405:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941428.898409:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941428.898417:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000040:1.0:1773941428.898419:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5ba0d64d00 count: 0 00000100:00000001:2.0:1773941428.898420:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941428.898422:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941428.898424:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941428.898426:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941428.898429:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941428.898437:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941428.898443:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941428.898447:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000010:3.0:1773941428.898449:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0300. 00000100:00000001:1.0:1773941428.898452:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1773941428.898455:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:1.0:1773941428.898460:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021c380 x1860112241146240/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941444 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941428.898465:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941428.898471:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34bb0 00000400:00000010:3.0:1773941428.898474:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34bb0. 00000100:00000001:3.0:1773941428.898477:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1773941428.898484:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941428.898487:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941428.898490:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce6400. 02000000:00000001:1.0:1773941428.898494:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941428.898496:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941428.898498:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941428.898500:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941428.898502:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b90e7fc00. 02000000:00000001:1.0:1773941428.898505:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941428.898506:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941428.898508:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021c380. 00000100:00000001:1.0:1773941428.898510:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941428.898511:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941428.898512:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941428.898514:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941428.898516:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941428.898519:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0880. 00000020:00000010:1.0:1773941428.898523:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7400. 00000020:00000010:1.0:1773941428.898527:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b7800. 00000040:00000010:1.0:1773941428.898530:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7a00. 00000020:00000010:1.0:1773941428.898535:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000100:00000200:3.0:1773941428.898536:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62d80 x1860112241146368/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941438 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941428.898539:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941428.898545:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941428.898551:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6e00. 00000100:00000001:3.0:1773941428.898552:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941428.898559:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62d80 x1860112241146368/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941438 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 20000000:00000010:1.0:1773941428.898559:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b7000. 10000000:00000040:1.0:1773941428.898564:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941428.898568:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941428.898571:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941428.898573:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b7800. 00000100:00000001:3.0:1773941428.898574:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000040:00000010:1.0:1773941428.898579:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7a00. 00000020:00000010:1.0:1773941428.898586:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941428.898590:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941428.898593:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941428.898597:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6e00. 20000000:00000010:1.0:1773941428.898598:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941428.898600:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7400. 10000000:00000010:1.0:1773941428.898602:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0880. 10000000:01000000:1.0:1773941428.898604:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941428.898606:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941428.898608:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000800:00000001:3.0:1773941428.899724:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941428.899729:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941428.899730:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941428.899732:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941428.899735:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941428.899738:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941428.899741:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941428.899743:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941428.899745:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941428.899748:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321603a00 00000400:00000200:3.0:1773941428.899750:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941428.899753:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xb5d [1] + 0 00000800:00000001:3.0:1773941428.899756:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941428.899789:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941428.899791:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941428.899800:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941428.899802:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941428.899805:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62d80 x1860112241146368/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941438 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941428.899811:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d62d80 x1860112241146368/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941438 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941428.899826:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941428.899830:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941428.899832:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941428.899850:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941428.899852:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941428.899854:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941428.899855:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941428.899857:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af98f0. 00000400:00000200:2.0:1773941428.899860:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34d48 00000400:00000010:2.0:1773941428.899865:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34d48. 00000100:00000001:2.0:1773941428.899867:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941428.899871:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62d80 x1860112241146368/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941438 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941428.899877:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62d80 x1860112241146368/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941438 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941428.899882:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941428.899883:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941428.899884:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941428.899885:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941428.899886:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941428.899887:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941428.899888:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941428.899891:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=7 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941428.899893:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941428.899894:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941428.899895:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941428.899897:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941428.899899:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941428.899900:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941428.899903:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62d80 x1860112241146368/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941438 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941428.899907:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941428.899908:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941428.899909:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941428.899910:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941428.899915:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941428.899916:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941428.899917:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941428.899919:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941428.899921:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941428.899923:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941428.899925:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941428.899929:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941428 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941428.899937:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941428.899944:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941428.899946:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941428.899948:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941428.899949:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941428.899951:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941428.899954:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941428.899957:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941428.899959:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941428.899962:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941428.899966:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941428.899968:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941428.900217:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941428.900221:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941428.900226:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62d80 x1860112241146368/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941438 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941428.900239:0:12155:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:0.0:1773941428.900243:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241146368:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941428.900247:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941428.900249:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1773941428.900251:0:12155:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000040:0.0:1773941428.900254:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62d80 x1860112241146368/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941438 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941428.900256:0:12155:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941428.900261:0:12155:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941428.900264:0:12155:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:0.0:1773941428.900264:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941428.900268:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:2.0:1773941428.900270:0:12155:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:0.0:1773941428.900270:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce6800. 00000001:00000001:2.0:1773941428.900274:0:12155:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:0.0:1773941428.900275:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941428.900277:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941428.900279:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941428.900280:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:2.0:1773941428.900282:0:12155:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941428.900282:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0735c00. 00000001:00000001:2.0:1773941428.900287:0:12155:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:0.0:1773941428.900287:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941428.900290:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941428.900291:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62d80. 00000100:00000001:0.0:1773941428.900293:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941428.900294:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941428.900295:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941428.900298:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000001:00000001:2.0:1773941428.900299:0:12155:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:0.0:1773941428.900301:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941428.900303:0:12155:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941428.900304:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000001:2.0:1773941428.900306:0:12155:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941428.900308:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941428.900310:0:12155:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941428.900316:0:12155:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941428.900319:0:12155:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941428.900321:0:12155:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941428.900324:0:12155:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941428.900326:0:12155:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941428.900328:0:12155:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941428.900330:0:12155:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941428.900331:0:12155:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:2.0:1773941428.900341:0:12155:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941428.900342:0:12155:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941428.900349:0:12155:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941428.900352:0:12155:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941428.900356:0:12155:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941428.900358:0:12155:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941428.900360:0:12155:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941428.900362:0:12155:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941428.900364:0:12155:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941428.902213:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941428.902217:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941428.902218:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941428.902221:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:1.1:1773941428.905379:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5ba0d64d00. 00010000:00000010:1.1:1773941428.905393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5bc2151e00. 00000020:00000001:1.0:1773941429.087822:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941429.087828:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941429.087832:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1773941430.111888:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941430.111896:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941430.111902:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:2.0:1773941430.175857:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941430.175863:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:2.0:1773941430.175868:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941430.175871:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941430.175873:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941430.175874:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941430.175877:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941430.175879:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941430.175881:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:2.0:1773941430.175884:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941430.175886:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941430.175894:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021d500. 00000020:00000040:2.0:1773941430.175898:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941430.175903:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941430.175906:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941430.175910:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941430.175917:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b90e7e000. 00000100:00000001:2.0:1773941430.175924:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941430.175928:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941430.175931:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941430.175938:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5ba0d64800. 00010000:00000001:2.0:1773941430.175943:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941430.175946:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5bc2151c00. 00000020:00000001:2.0:1773941430.175949:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941430.175951:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5bc2151c00 with handle 0xa33a5be14eae1c75 to hash 00000020:00000001:2.0:1773941430.175953:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941430.175954:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615824947289088 : -128248762262528 : ffff8b5bc2151c00) 00010000:00000001:2.0:1773941430.175957:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615824947289088 : -128248762262528 : ffff8b5bc2151c00) 00010000:00000001:2.0:1773941430.175961:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941430.175977:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941430.175981:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151c00/0xa33a5be14eae1c75 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941430.175990:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151c00/0xa33a5be14eae1c75 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941430.176001:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151c00/0xa33a5be14eae1c75 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941430.176005:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941430.176007:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941430.176013:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b543c0. 00000100:00000001:2.0:1773941430.176017:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387359680 : -128249322191936 : ffff8b5ba0b543c0) 00000100:00000001:2.0:1773941430.176021:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941430.176022:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109262080 : -128249600289536 : ffff8b5b9021d500) 00000100:00000001:2.0:1773941430.176028:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941430.176033:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941430.176035:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941430.176038:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941430.176050:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021d500 x1860112241146496/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941430.176058:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941430.176060:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941430.176063:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941430.176066:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941430.176069:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941430.176073:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021d500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241146496:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941430.176076:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941430.176081:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941430.176084:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941430.176086:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941430.176111:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce7000. 02000000:00000001:2.0:1773941430.176117:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941430.176127:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941430.176131:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af98f0. 00000400:00000010:2.0:1773941430.176140:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34d48. 00000100:00000200:2.0:1773941430.176147:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241146496, portal 25 00000100:00000001:2.0:1773941430.176149:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941430.176150:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109262080 : -128249600289536 : ffff8b5b9021d500) 00000100:00000040:2.0:1773941430.176155:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021d500 x1860112241146496/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941446 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941430.176160:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941430.176163:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941430.176164:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f347f8. 00000100:00000200:2.0:1773941430.176166:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241146496, offset 0 00000400:00000200:2.0:1773941430.176171:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941430.176179:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941430.176185:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941430.176190:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941430.176194:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941430.176197:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941430.176199:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941430.176201:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[18, 17] 00000400:00000200:2.0:1773941430.176204:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:2.0:1773941430.176207:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [124:124:256:4294967295] 192.168.204.153@tcp LPNI seq info [19:19:8:4294967295] 00000400:00000200:2.0:1773941430.176210:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941430.176217:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:2.0:1773941430.176222:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:2.0:1773941430.176226:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0300. 00000800:00000200:2.0:1773941430.176232:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:2.0:1773941430.176238:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:2.0:1773941430.176242:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0300 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941430.176254:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941430.176260:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941430.176263:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294870539 00000100:00000001:2.0:1773941430.176274:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294869261 00000100:00000001:2.0:1773941430.176278:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294870539 00000100:00000001:2.0:1773941430.176281:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294869261 00000100:00000001:2.0:1773941430.176283:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941430.176285:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941430.176287:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941430.176289:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941430.176292:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b543c0 going to sleep for 11 seconds 00000100:00000001:2.0:1773941430.176294:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941430.176297:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941430.176305:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941430.176309:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0300. 00000400:00000200:3.0:1773941430.176315:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941430.176320:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941430.176324:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f347f8 00000400:00000010:3.0:1773941430.176326:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f347f8. 00000100:00000001:3.0:1773941430.176335:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941430.176339:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021d500 x1860112241146496/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941446 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941430.176348:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941430.176350:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b9021d500 x1860112241146496/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941446 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941430.176355:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941430.176356:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941430.177136:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941430.177142:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941430.177144:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941430.177146:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941430.177150:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941430.177153:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941430.177158:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941430.177160:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941430.177190:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941430.177193:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321603a80 00000400:00000200:3.0:1773941430.177197:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941430.177200:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xb6d [1] + 224 00000800:00000001:3.0:1773941430.177205:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941430.177210:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941430.177212:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941430.177218:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941430.177222:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941430.177225:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021d500 x1860112241146496/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941446 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941430.177232:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b9021d500 x1860112241146496/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941446 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941430.177243:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941430.177248:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941430.177251:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941430.177262:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941430.177267:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941430.177269:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941430.177270:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af98f0. 00000400:00000200:2.0:1773941430.177273:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34d48 00000400:00000010:2.0:1773941430.177274:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34d48. 00000100:00000001:2.0:1773941430.177276:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941430.177280:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021d500 x1860112241146496/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941446 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941430.177288:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021d500 x1860112241146496/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941446 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941430.177292:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941430.177293:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941430.177295:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941430.177297:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941430.177299:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941430.177301:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941430.177302:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941430.177307:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=9 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941430.177309:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941430.177311:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941430.177315:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:2.0:1773941430.177321:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941430.177323:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941430.177326:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941430.177334:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941430.177338:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941430.177340:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941430.177342:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941430.177343:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941430.177345:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941430.177347:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b9021d500 x1860112241146496/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941446 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941430.177352:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941430.177353:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941430.177354:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941430.177356:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941430.177359:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941430.177360:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941430.177361:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941430.177361:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941430.177362:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941430.177363:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941430.177364:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941430.177366:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941430.177367:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941430.177370:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941430.177371:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 199 00000400:00000200:2.0:1773941430.177374:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941430.177377:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941430.177379:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941430.177381:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941430.177383:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941430.177385:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:2.0:1773941430.177392:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941430.177394:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941430.177396:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941430.177399:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941430.177401:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941430.177403:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:2.0:1773941430.177413:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941430.177415:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941430.177416:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941430.177418:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:2.0:1773941430.177422:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:2.0:1773941430.177425:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:2.0:1773941430.177428:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941430.177431:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941430.177434:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941430.177436:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941430.177438:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941430.177440:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941430.177442:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021ca80. 00000020:00000040:2.0:1773941430.177444:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:2.0:1773941430.177446:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941430.177448:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941430.177449:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941430.177451:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941430.177452:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941430.177455:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0ce4c00. 00000100:00000001:2.0:1773941430.177457:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941430.177461:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b9021ca80 x1860112241146624/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941430.177466:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941430.177468:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941430.177471:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000005f3b654] to pc [ptlrpcd_rcv+-1] req@ffff8b5b9021ca80 x1860112241146624/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941430.177489:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941430.177491:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941430.177492:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941430.177494:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941430.177496:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941430.177497:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941430.177498:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b543c0 going to sleep for 1 seconds 00000100:00000001:2.0:1773941430.177500:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941430.177501:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941430.177506:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021d500 x1860112241146496/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941446 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941430.177511:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941430.177513:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941430.177513:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941430.177517:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941430.177518:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941430.177519:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941430.177520:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941430.177524:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021d500 x1860112241146496/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941446 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941430.177528:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021ca80 x1860112241146624/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941430.177533:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021d500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241146496:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941430.177541:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941430.177545:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941430.177545:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941430.177548:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941430.177549:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941430.177550:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941430.177553:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021d500 x1860112241146496/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941446 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1773941430.177560:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1773941430.177561:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941430.177565:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b543c0. 02000000:00000001:0.0:1773941430.177565:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941430.177568:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:0.0:1773941430.177571:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941430.177572:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:2.0:1773941430.177575:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00100000:0.0:1773941430.177579:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241146624:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:2.0:1773941430.177585:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:0.0:1773941430.177586:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1773941430.177588:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941430.177590:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5bc2151c00 refcount=4 02000000:00000001:0.0:1773941430.177590:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:2.0:1773941430.177592:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824947289088 : -128248762262528 : ffff8b5bc2151c00) 00010000:00000001:2.0:1773941430.177594:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615824947289088 : -128248762262528 : ffff8b5bc2151c00) 00010000:00010000:2.0:1773941430.177596:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151c00/0xa33a5be14eae1c75 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941430.177603:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:0.0:1773941430.177603:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941430.177609:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00010000:2.0:1773941430.177610:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151c00/0xa33a5be14eae1c75 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941430.177616:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000010:0.0:1773941430.177616:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0000. 00010000:00010000:2.0:1773941430.177619:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151c00/0xa33a5be14eae1c75 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941430.177621:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941430.177627:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941430.177629:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:0.0:1773941430.177630:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:2.0:1773941430.177631:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151c00/0xa33a5be14eae1c75 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941430.177634:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00000001:2.0:1773941430.177638:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941430.177640:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1773941430.177640:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00010000:00000001:2.0:1773941430.177642:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000200:0.0:1773941430.177644:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241146624, portal 25 00010000:00010000:2.0:1773941430.177645:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151c00/0xa33a5be14eae1c75 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941430.177648:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941430.177651:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109259392 : -128249600292224 : ffff8b5b9021ca80) 00010000:00010000:2.0:1773941430.177653:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5bc2151c00 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:2.0:1773941430.177655:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941430.177658:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151c00/0xa33a5be14eae1c75 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941430.177658:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021ca80 x1860112241146624/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941440 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941430.177672:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941430.177677:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941430.177681:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c5d8. 00010000:00000001:2.0:1773941430.177683:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000200:0.0:1773941430.177684:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241146624, offset 0 00010000:00000001:2.0:1773941430.177685:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941430.177687:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941430.177688:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5bc2151c00 refcount=5 00000400:00000200:0.0:1773941430.177691:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000020:00000001:2.0:1773941430.177692:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824947289088 : -128248762262528 : ffff8b5bc2151c00) 00010000:00000001:2.0:1773941430.177695:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615824947289088 : -128248762262528 : ffff8b5bc2151c00) 00000400:00000200:0.0:1773941430.177697:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:2.0:1773941430.177699:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:0.0:1773941430.177701:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00010000:2.0:1773941430.177702:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151c00/0xa33a5be14eae1c75 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:2.0:1773941430.177730:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941430.177732:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151c00/0xa33a5be14eae1c75 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941430.177736:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:01000000:2.0:1773941430.177739:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:0.0:1773941430.177741:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 10000000:00000040:2.0:1773941430.177743:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:0.0:1773941430.177744:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000001:2.0:1773941430.177746:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941430.177749:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:2.0:1773941430.177753:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941430.177757:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:0.0:1773941430.177757:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[18, 19] 00010000:00000001:2.0:1773941430.177762:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:0.0:1773941430.177762:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:2.0:1773941430.177765:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:0.0:1773941430.177767:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [125:125:256:4294967295] 192.168.204.143@tcp LPNI seq info [20:20:8:4294967295] 00010000:00000001:2.0:1773941430.177768:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941430.177769:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941430.177771:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5bc2151c00 with handle 0xa33a5be14eae1c75 from hash 00010000:00000001:2.0:1773941430.177772:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941430.177773:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941430.177775:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941430.177777:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941430.177778:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941430.177779:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:2.0:1773941430.177780:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941430.177781:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941430.177783:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941430.177786:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:2.0:1773941430.177790:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941430.177794:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151c00/0xa33a5be14eae1c75 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941430.177796:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941430.177805:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:2.0:1773941430.177806:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941430.177809:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941430.177810:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000800:00000010:0.0:1773941430.177810:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00010000:00000001:2.0:1773941430.177811:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941430.177814:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941430.177815:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941430.177817:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:0.0:1773941430.177819:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00010000:2.0:1773941430.177820:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5bc2151c00/0xa33a5be14eae1c75 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:2.0:1773941430.177826:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5ba0d64800 count: 0 00000800:00000200:0.0:1773941430.177828:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941430.177831:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:2.0:1773941430.177835:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941430.177837:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941430.177838:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941430.177840:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941430.177842:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021d500 x1860112241146496/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941446 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941430.177847:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941430.177851:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294870541 00000100:00000001:2.0:1773941430.177854:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941430.177856:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294870539 02000000:00000001:2.0:1773941430.177857:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941430.177861:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce7000. 00000100:00000001:0.0:1773941430.177863:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294870541 02000000:00000001:2.0:1773941430.177864:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941430.177867:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941430.177868:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294870539 00000020:00000040:2.0:1773941430.177869:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941430.177871:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1773941430.177872:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941430.177874:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941430.177876:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b90e7e000. 00000100:00000001:0.0:1773941430.177877:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941430.177879:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 02000000:00000001:2.0:1773941430.177880:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1773941430.177881:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941430.177883:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000800:00000010:3.0:1773941430.177884:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000100:00000010:2.0:1773941430.177886:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021d500. 00000100:00000001:0.0:1773941430.177888:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:3.0:1773941430.177889:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941430.177890:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941430.177890:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941430.177893:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941430.177896:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941430.177896:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1773941430.177899:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:0.0:1773941430.177899:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:01000000:2.0:1773941430.177900:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:0.0:1773941430.177903:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941430.177905:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000400:00000200:3.0:1773941430.177906:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c5d8 00000100:00000001:0.0:1773941430.177906:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941430.177910:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c5d8. 00000100:00000001:3.0:1773941430.177913:0:2347:0:(events.c:34:request_out_callback()) Process entered 10000000:00000010:2.0:1773941430.177913:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b845ad7c0. 00000020:00000010:2.0:1773941430.177918:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b90e7e000. 00000020:00000010:2.0:1773941430.177922:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b90e7f200. 00000040:00000010:2.0:1773941430.177925:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b90e7f400. 00000020:00000010:2.0:1773941430.177929:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b84e45800. 00000020:00000010:2.0:1773941430.177932:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b90e7e400. 80000000:00000010:2.0:1773941430.177935:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba21bdf60. 00040000:00000010:2.0:1773941430.177939:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b90e7fc00. 20000000:00000010:2.0:1773941430.177943:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b90e7f800. 10000000:00000040:2.0:1773941430.177947:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:2.0:1773941430.177948:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000200:3.0:1773941430.177952:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021ca80 x1860112241146624/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941440 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1773941430.177953:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b90e7f200. 00000040:00000010:2.0:1773941430.177958:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b90e7f400. 00000020:00000010:2.0:1773941430.177961:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b84e45800. 00000020:00000010:2.0:1773941430.177966:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b90e7e400. 00000100:00000001:3.0:1773941430.177970:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 80000000:00000010:2.0:1773941430.177971:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba21bdf60. 00040000:00000010:2.0:1773941430.177975:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b90e7fc00. 00000100:00000040:3.0:1773941430.177977:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021ca80 x1860112241146624/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941440 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 20000000:00000010:2.0:1773941430.177978:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b90e7f800. 00000020:00000010:2.0:1773941430.177981:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b90e7e000. 10000000:00000010:2.0:1773941430.177985:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b845ad7c0. 10000000:01000000:2.0:1773941430.177988:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000100:00000001:3.0:1773941430.177990:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941430.177990:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941430.177992:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:00020000:2.0:1773941430.177994:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941430.177996:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941430.177997:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941430.177999:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:2.0:1773941430.178000:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941430.179370:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941430.179374:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941430.179375:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941430.179377:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941430.179379:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941430.179382:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941430.179385:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941430.179387:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941430.179389:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941430.179395:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321603b00 00000400:00000200:3.0:1773941430.179398:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941430.179428:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xb7d [1] + 0 00000800:00000001:3.0:1773941430.179431:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941430.179435:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941430.179437:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941430.179439:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941430.179442:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941430.179444:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021ca80 x1860112241146624/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941440 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941430.179456:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b9021ca80 x1860112241146624/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941440 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941430.179465:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941430.179468:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941430.179470:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941430.179488:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941430.179490:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941430.179493:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941430.179494:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941430.179495:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941430.179498:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941430.179499:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941430.179501:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941430.179505:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021ca80 x1860112241146624/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941440 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941430.179521:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021ca80 x1860112241146624/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941440 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941430.179526:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941430.179527:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941430.179528:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941430.179530:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941430.179530:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941430.179532:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941430.179533:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941430.179535:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=9 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941430.179537:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941430.179539:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941430.179540:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941430.179542:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941430.179543:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941430.179545:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941430.179548:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021ca80 x1860112241146624/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941440 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941430.179552:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941430.179553:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941430.179554:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941430.179556:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941430.179559:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941430.179560:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941430.179561:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941430.179563:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941430.179564:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941430.179566:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941430.179567:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941430.179570:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941430 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941430.179571:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941430.179576:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941430.179581:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941430.179582:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941430.179584:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941430.179585:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941430.179588:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941430.179589:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941430.179591:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941430.179594:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941430.179596:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941430.179598:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941430.179758:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941430.179761:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941430.179765:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021ca80 x1860112241146624/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941440 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941430.179769:0:12157:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941430.179772:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241146624:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00080000:0.0:1773941430.179777:0:12157:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941430.179783:0:12157:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941430.179784:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941430.179785:0:12157:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941430.179789:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:01000000:0.0:1773941430.179789:0:12157:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941430.179793:0:12157:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000040:2.0:1773941430.179794:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021ca80 x1860112241146624/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941440 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:0.0:1773941430.179796:0:12157:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941430.179801:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941430.179801:0:12157:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941430.179804:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941430.179805:0:12157:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941430.179806:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0000. 00000001:00000001:0.0:1773941430.179807:0:12157:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941430.179810:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941430.179810:0:12157:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941430.179813:0:2351:0:(genops.c:1236:class_import_put()) Process entered 10000000:00000001:0.0:1773941430.179813:0:12157:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941430.179815:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941430.179816:0:12157:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:2.0:1773941430.179819:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941430.179821:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0ce4c00. 00000100:00080000:0.0:1773941430.179821:0:12157:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941430.179824:0:12157:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:2.0:1773941430.179825:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1773941430.179827:0:12157:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:2.0:1773941430.179828:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941430.179831:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021ca80. 00000100:00000001:0.0:1773941430.179831:0:12157:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941430.179836:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:0.0:1773941430.179836:0:12157:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941430.179838:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941430.179841:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941430.179841:0:12157:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941430.179843:0:12157:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:2.0:1773941430.179846:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000040:0.0:1773941430.179846:0:12157:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:2.0:1773941430.179849:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941430.179852:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941430.179853:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941430.179861:0:12157:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941430.179863:0:12157:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941430.179864:0:12157:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941430.179868:0:12157:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941430.179882:0:12157:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941430.179886:0:12157:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941430.179889:0:12157:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941430.179892:0:12157:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941430.179893:0:12157:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941430.186735:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5ba0d64800. 00010000:00000010:2.1:1773941430.186743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5bc2151c00. 00000020:00000001:3.0:1773941431.135893:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941431.135901:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941431.135905:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941431.775866:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941431.775876:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941431.775881:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941431.775883:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941431.775885:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941431.775886:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941431.775890:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941431.775891:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941431.775894:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941431.775897:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941431.775898:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941431.775920:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62a00. 00000020:00000040:1.0:1773941431.775926:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941431.775931:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941431.775935:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941431.775941:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941431.775948:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7400. 00000100:00000001:1.0:1773941431.775956:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941431.775960:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941431.775979:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941431.775988:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ecd00. 00010000:00000001:1.0:1773941431.775994:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941431.775997:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f000. 00000020:00000001:1.0:1773941431.776001:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941431.776004:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f000 with handle 0xa33a5be14eae1c7c to hash 00000020:00000001:1.0:1773941431.776006:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941431.776007:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941431.776010:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941431.776014:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941431.776017:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941431.776020:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c7c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941431.776029:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c7c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941431.776040:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c7c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941431.776045:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941431.776047:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941431.776053:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6c00. 00000100:00000001:1.0:1773941431.776055:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258984960 : -128249450566656 : ffff8b5b990e6c00) 00000100:00000001:1.0:1773941431.776070:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941431.776072:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121080320 : -128249588471296 : ffff8b5b90d62a00) 00000100:00000001:1.0:1773941431.776078:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941431.776083:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941431.776085:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941431.776086:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941431.776098:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62a00 x1860112241146752/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941431.776105:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941431.776113:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941431.776117:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941431.776119:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941431.776121:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941431.776124:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241146752:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941431.776128:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941431.776134:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941431.776136:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941431.776138:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941431.776142:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735400. 02000000:00000001:1.0:1773941431.776144:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941431.776152:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941431.776155:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941431.776162:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941431.776170:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241146752, portal 25 00000100:00000001:1.0:1773941431.776171:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941431.776172:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121080320 : -128249588471296 : ffff8b5b90d62a00) 00000100:00000040:1.0:1773941431.776177:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62a00 x1860112241146752/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941447 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941431.776183:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941431.776185:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941431.776187:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1aa0. 00000100:00000200:1.0:1773941431.776190:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241146752, offset 0 00000400:00000200:1.0:1773941431.776196:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941431.776205:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941431.776214:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941431.776220:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941431.776224:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941431.776227:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941431.776229:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941431.776232:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[20, 19] 00000400:00000200:1.0:1773941431.776234:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941431.776238:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [126:126:256:4294967295] 192.168.204.153@tcp LPNI seq info [21:21:8:4294967295] 00000400:00000200:1.0:1773941431.776245:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941431.776255:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941431.776260:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941431.776264:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b400. 00000800:00000200:1.0:1773941431.776272:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941431.776280:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941431.776286:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b400 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941431.776310:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941431.776315:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941431.776318:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294872139 00000100:00000001:1.0:1773941431.776326:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294870541 00000100:00000001:1.0:1773941431.776333:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294872139 00000100:00000001:1.0:1773941431.776338:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294870541 00000100:00000001:1.0:1773941431.776343:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941431.776347:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941431.776350:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941431.776353:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941431.776357:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6c00 going to sleep for 11 seconds 00000100:00000001:1.0:1773941431.776360:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941431.776365:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941431.776463:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941431.776468:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b400. 00000400:00000200:3.0:1773941431.776475:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941431.776480:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941431.776486:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1aa0 00000400:00000010:3.0:1773941431.776488:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1aa0. 00000100:00000001:3.0:1773941431.776492:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941431.776496:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62a00 x1860112241146752/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941447 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941431.776506:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941431.776509:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d62a00 x1860112241146752/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941447 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941431.776515:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941431.776516:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941431.777334:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941431.777348:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941431.777352:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941431.777354:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941431.777360:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941431.777407:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941431.777413:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941431.777417:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941431.777421:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941431.777425:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321603b80 00000400:00000200:3.0:1773941431.777428:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941431.777432:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xb85 [1] + 224 00000800:00000001:3.0:1773941431.777438:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941431.777445:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941431.777447:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941431.777449:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941431.777453:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941431.777457:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62a00 x1860112241146752/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941447 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941431.777465:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d62a00 x1860112241146752/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941447 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941431.777479:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941431.777486:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941431.777488:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941431.777514:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941431.777521:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941431.777522:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941431.777525:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941431.777529:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941431.777530:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941431.777532:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941431.777537:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62a00 x1860112241146752/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941447 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941431.777545:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62a00 x1860112241146752/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941447 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941431.777561:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941431.777563:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941431.777567:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941431.777570:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941431.777573:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941431.777576:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941431.777579:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941431.777587:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=10 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941431.777592:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941431.777595:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941431.777601:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941431.781449:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941431.781454:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941431.781458:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941431.781463:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941431.781468:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941431.785836:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941431.785839:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941431.785841:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941431.785843:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941431.785850:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d62a00 x1860112241146752/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941447 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941431.785858:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941431.785859:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941431.785861:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941431.785865:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941431.785868:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941431.785870:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941431.785872:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941431.785873:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941431.785874:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941431.785876:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941431.785891:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941431.785894:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941431.785896:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941431.785899:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941431.785902:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 200 00000400:00000200:1.0:1773941431.785908:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941431.785914:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941431.785918:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941431.785921:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941431.785923:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941431.785925:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941431.785928:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941431.785930:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941431.785933:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941431.785935:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941431.785937:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941431.785939:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941431.785941:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941431.785943:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941431.785944:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941431.785945:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941431.785948:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941431.785950:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941431.785952:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941431.785954:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941431.785958:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941431.785960:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941431.785962:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941431.785963:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941431.785968:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63100. 00000020:00000040:1.0:1773941431.785971:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941431.785975:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941431.785980:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941431.785982:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941431.785986:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941431.785989:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941431.785996:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737c00. 00000100:00000001:1.0:1773941431.786001:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941431.786006:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63100 x1860112241146880/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941431.786014:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941431.786018:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941431.786022:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000025727364] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63100 x1860112241146880/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941431.786041:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941431.786044:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941431.786045:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941431.786049:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941431.786052:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941431.786055:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941431.786071:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6c00 going to sleep for 1 seconds 00000100:00000001:1.0:1773941431.786076:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941431.786079:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941431.786082:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1773941431.786087:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62a00 x1860112241146752/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941447 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941431.786088:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941431.786092:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941431.786101:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63100 x1860112241146880/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941431.786104:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941431.786113:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941431.786116:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941431.786118:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941431.786119:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941431.786121:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941431.786124:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941431.786127:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1773941431.786129:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62a00 x1860112241146752/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941447 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941431.786130:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941431.786139:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241146880:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941431.786147:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1773941431.786148:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241146752:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941431.786153:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941431.786160:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941431.786163:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941431.786164:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941431.786168:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:2.0:1773941431.786169:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941431.786172:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000010:2.0:1773941431.786176:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce4c00. 00000100:00000040:1.0:1773941431.786177:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62a00 x1860112241146752/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941447 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941431.786181:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941431.786188:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941431.786193:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941431.786195:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6c00. 00000100:00000001:1.0:1773941431.786200:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000400:00000010:2.0:1773941431.786202:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af98f0. 00000100:00000001:1.0:1773941431.786202:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941431.786208:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941431.786212:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000010:2.0:1773941431.786213:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34d48. 00000020:00000001:1.0:1773941431.786214:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941431.786218:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=4 00000020:00000001:1.0:1773941431.786222:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000100:00000200:2.0:1773941431.786224:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241146880, portal 25 00010000:00000001:1.0:1773941431.786227:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000100:00000001:2.0:1773941431.786229:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941431.786233:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121082112 : -128249588469504 : ffff8b5b90d63100) 00010000:00010000:1.0:1773941431.786237:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c7c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941431.786243:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63100 x1860112241146880/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941431.786256:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941431.786262:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:1.0:1773941431.786266:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c7c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941431.786267:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941431.786273:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34f68. 00000100:00000200:2.0:1773941431.786279:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241146880, offset 0 00010000:00000001:1.0:1773941431.786281:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941431.786285:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c7c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941431.786289:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941431.786297:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941431.786300:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941431.786303:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941431.786304:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c7c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941431.786310:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941431.786318:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941431.786319:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941431.786323:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941431.786328:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941431.786331:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941431.786333:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941431.786334:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[20, 21] 00010000:00000001:1.0:1773941431.786341:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941431.786342:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00010000:1.0:1773941431.786347:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c7c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941431.786349:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [127:127:256:4294967295] 192.168.204.143@tcp LPNI seq info [22:22:8:4294967295] 00000400:00000200:2.0:1773941431.786364:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941431.786368:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f000 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941431.786373:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941431.786377:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c7c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941431.786383:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941431.786387:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941431.786390:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941431.786391:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941431.786394:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=5 00000800:00000200:2.0:1773941431.786395:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000020:00000001:1.0:1773941431.786396:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941431.786401:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941431.786404:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000010:2.0:1773941431.786405:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0500. 00010000:00010000:1.0:1773941431.786407:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c7c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:1.0:1773941431.786413:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:2.0:1773941431.786414:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00010000:1.0:1773941431.786417:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c7c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941431.786423:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941431.786426:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941431.786428:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941431.786431:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000800:00000200:2.0:1773941431.786434:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941431.786435:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941431.786439:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000800:00000200:2.0:1773941431.786440:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0500 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941431.786442:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941431.786446:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941431.786447:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941431.786449:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f000 with handle 0xa33a5be14eae1c7c from hash 00000100:00000001:2.0:1773941431.786457:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941431.786463:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:2.0:1773941431.786464:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941431.786466:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941431.786468:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294872149 00010000:00000001:1.0:1773941431.786469:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941431.786473:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941431.786476:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941431.786478:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941431.786480:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941431.786481:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294872139 00010000:00000001:1.0:1773941431.786482:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941431.786485:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941431.786488:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941431.786490:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294872149 00010000:00010000:1.0:1773941431.786492:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c7c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941431.786497:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294872139 00010000:00000001:1.0:1773941431.786498:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941431.786500:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941431.786504:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:2.0:1773941431.786505:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941431.786506:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941431.786508:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941431.786512:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941431.786513:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941431.786515:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941431.786519:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941431.786519:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c7c lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941431.786527:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:1.0:1773941431.786529:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ecd00 count: 0 00000100:00000001:2.0:1773941431.786538:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941431.786541:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941431.786545:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1773941431.786546:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941431.786547:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941431.786549:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941431.786552:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941431.786553:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941431.786557:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941431.786559:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62a00 x1860112241146752/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941447 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1773941431.786566:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1773941431.786570:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000800:00000010:3.0:1773941431.786572:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0500. 02000000:00000001:1.0:1773941431.786577:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:3.0:1773941431.786580:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:1.0:1773941431.786582:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735400. 00000400:00000200:3.0:1773941431.786587:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 02000000:00000001:1.0:1773941431.786591:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:3.0:1773941431.786593:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34f68 00000020:00000001:1.0:1773941431.786595:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000010:3.0:1773941431.786597:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34f68. 00000020:00000040:1.0:1773941431.786600:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941431.786603:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1773941431.786605:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000200:3.0:1773941431.786608:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63100 x1860112241146880/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1773941431.786617:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7400. 02000000:00000001:1.0:1773941431.786624:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941431.786627:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941431.786630:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62a00. 00000100:00000001:3.0:1773941431.786633:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941431.786635:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941431.786638:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941431.786639:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63100 x1860112241146880/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941441 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941431.786641:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941431.786647:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941431.786652:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941431.786657:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0b00. 00000020:00000010:1.0:1773941431.786661:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7400. 00000020:00000010:1.0:1773941431.786665:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b7000. 00000040:00000010:1.0:1773941431.786669:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941431.786673:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941431.786677:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941431.786682:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941431.786686:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7a00. 20000000:00000010:1.0:1773941431.786690:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b7800. 10000000:00000040:1.0:1773941431.786693:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941431.786695:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941431.786700:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b7000. 00000040:00000010:1.0:1773941431.786744:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6e00. 00000100:00000001:3.0:1773941431.786747:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941431.786748:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000100:00000001:3.0:1773941431.786751:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1773941431.786755:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941431.786760:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941431.786765:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7a00. 20000000:00000010:1.0:1773941431.786769:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941431.786772:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7400. 10000000:00000010:1.0:1773941431.786775:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0b00. 10000000:01000000:1.0:1773941431.786778:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941431.786781:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941431.786784:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941431.786786:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941431.786788:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941431.786794:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941431.786796:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941431.791224:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941431.791243:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941431.791247:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941431.791249:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941431.791255:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941431.791260:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941431.791265:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941431.791268:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941431.791272:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941431.791276:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321603c00 00000400:00000200:3.0:1773941431.791279:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941431.791283:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xb8d [1] + 0 00000800:00000001:3.0:1773941431.791289:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941431.791296:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941431.791298:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941431.791302:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941431.791351:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941431.791359:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63100 x1860112241146880/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941441 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941431.791373:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63100 x1860112241146880/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941441 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941431.791395:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941431.791402:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941431.791406:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941431.791424:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941431.791428:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941431.791433:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941431.791434:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941431.791437:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af98f0. 00000400:00000200:2.0:1773941431.791440:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34d48 00000400:00000010:2.0:1773941431.791442:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34d48. 00000100:00000001:2.0:1773941431.791444:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941431.791449:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63100 x1860112241146880/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941441 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941431.791457:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63100 x1860112241146880/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941441 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941431.791464:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941431.791465:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941431.791467:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941431.791478:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941431.791480:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941431.791482:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941431.791484:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941431.791490:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=10 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941431.791494:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941431.791498:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941431.791500:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941431.791503:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941431.791505:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941431.791508:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941431.791514:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63100 x1860112241146880/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941441 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941431.791523:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941431.791525:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941431.791527:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941431.791530:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941431.791538:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941431.791541:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941431.791542:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941431.791545:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941431.791547:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941431.791549:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941431.791552:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941431.791557:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941431 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941431.791560:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941431.791566:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941431.791567:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941431.791569:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941431.791571:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941431.791572:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941431.791576:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941431.791578:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941431.791580:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941431.791588:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941431.791592:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941431.791594:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00010000:00000010:1.1:1773941431.791749:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ecd00. 00010000:00000010:1.1:1773941431.791759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f000. 00000100:00000001:2.0:1773941431.791867:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941431.791871:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941431.791876:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63100 x1860112241146880/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941441 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941431.791883:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241146880:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941431.791887:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941431.791889:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941431.791890:0:12158:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941431.791893:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63100 x1860112241146880/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941441 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941431.791900:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941431.791902:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941431.791906:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce4c00. 00000100:00080000:0.0:1773941431.791906:0:12158:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 02000000:00000001:2.0:1773941431.791912:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941431.791914:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00100000:0.0:1773941431.791915:0:12158:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000040:2.0:1773941431.791916:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941431.791920:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941431.791921:0:12158:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:2.0:1773941431.791923:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737c00. 10000000:01000000:0.0:1773941431.791926:0:12158:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941431.791927:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941431.791930:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941431.791932:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63100. 00000001:00000001:0.0:1773941431.791933:0:12158:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941431.791935:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941431.791938:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:0.0:1773941431.791940:0:12158:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941431.791941:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941431.791947:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941431.791949:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941431.791949:0:12158:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941431.791954:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000001:00000001:0.0:1773941431.791956:0:12158:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941431.791957:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941431.791960:0:12158:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941431.791963:0:12158:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941431.791966:0:12158:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941431.791969:0:12158:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941431.791975:0:12158:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941431.791978:0:12158:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941431.791981:0:12158:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941431.791983:0:12158:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941431.791986:0:12158:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941431.791989:0:12158:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941431.791991:0:12158:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941431.791993:0:12158:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941431.792012:0:12158:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941431.792015:0:12158:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941431.792018:0:12158:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941431.792023:0:12158:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941431.798549:0:12158:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941431.798555:0:12158:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941431.798558:0:12158:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941431.798561:0:12158:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941431.798563:0:12158:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941432.159827:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941432.159834:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941432.159839:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941432.223821:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (207) 00000020:00000001:3.0:1773941433.183898:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941433.183904:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941433.183926:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941433.695856:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941433.695867:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941433.695873:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941433.695877:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941433.695878:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941433.695880:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941433.695884:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941433.695886:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941433.695889:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941433.695893:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941433.695895:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941433.695911:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62a00. 00000020:00000040:1.0:1773941433.695918:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941433.695925:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941433.695929:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941433.695936:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941433.695945:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7400. 00000100:00000001:1.0:1773941433.695954:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941433.695960:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941433.695964:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941433.695975:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ecd00. 00010000:00000001:1.0:1773941433.695982:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941433.695987:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f000. 00000020:00000001:1.0:1773941433.695993:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941433.695996:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f000 with handle 0xa33a5be14eae1c83 to hash 00000020:00000001:1.0:1773941433.695999:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941433.696001:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941433.696016:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941433.696038:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941433.696042:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941433.696048:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c83 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941433.696087:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c83 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941433.696102:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c83 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941433.696110:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941433.696112:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941433.696121:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6c00. 00000100:00000001:1.0:1773941433.696124:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258984960 : -128249450566656 : ffff8b5b990e6c00) 00000100:00000001:1.0:1773941433.696128:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941433.696130:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121080320 : -128249588471296 : ffff8b5b90d62a00) 00000100:00000001:1.0:1773941433.696137:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941433.696142:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941433.696145:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941433.696147:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941433.696161:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62a00 x1860112241147008/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941433.696173:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941433.696175:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941433.696180:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941433.696184:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941433.696187:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941433.696191:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241147008:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941433.696197:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941433.696212:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941433.696215:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941433.696218:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941433.696224:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735400. 02000000:00000001:1.0:1773941433.696227:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941433.696237:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941433.696248:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941433.696258:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941433.696268:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241147008, portal 25 00000100:00000001:1.0:1773941433.696271:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941433.696273:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121080320 : -128249588471296 : ffff8b5b90d62a00) 00000100:00000040:1.0:1773941433.696280:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62a00 x1860112241147008/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941449 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941433.696289:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941433.696292:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941433.696295:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f13b8. 00000100:00000200:1.0:1773941433.696300:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241147008, offset 0 00000400:00000200:1.0:1773941433.696307:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941433.696318:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941433.696328:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941433.696335:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941433.696340:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941433.696345:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941433.696349:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941433.696363:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[22, 21] 00000400:00000200:1.0:1773941433.696368:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941433.696374:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [128:128:256:4294967295] 192.168.204.153@tcp LPNI seq info [23:23:8:4294967295] 00000400:00000200:1.0:1773941433.696379:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941433.696393:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941433.696401:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941433.696408:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024bf00. 00000800:00000200:1.0:1773941433.696416:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941433.696424:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941433.696429:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024bf00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941433.696446:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941433.696449:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941433.696452:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294874059 00000100:00000001:1.0:1773941433.696464:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294872149 00000100:00000001:1.0:1773941433.696469:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294874059 00000100:00000001:1.0:1773941433.696471:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294872149 00000100:00000001:1.0:1773941433.696474:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941433.696477:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941433.696479:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941433.696481:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941433.696484:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6c00 going to sleep for 11 seconds 00000100:00000001:1.0:1773941433.696487:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941433.696491:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941433.696596:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941433.696603:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024bf00. 00000400:00000200:3.0:1773941433.696610:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941433.696618:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941433.696626:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f13b8 00000400:00000010:3.0:1773941433.696629:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f13b8. 00000100:00000001:3.0:1773941433.696634:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941433.696643:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62a00 x1860112241147008/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941449 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941433.696657:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941433.696661:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d62a00 x1860112241147008/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941449 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941433.696668:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941433.696670:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941433.697798:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941433.697810:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941433.697813:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941433.697816:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941433.697823:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941433.697830:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941433.697838:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941433.697842:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941433.697848:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941433.697852:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321603c80 00000400:00000200:3.0:1773941433.697857:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941433.697864:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xb9d [1] + 224 00000800:00000001:3.0:1773941433.697916:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941433.697925:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941433.697928:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941433.697932:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941433.697936:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941433.697942:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62a00 x1860112241147008/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941449 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941433.697967:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d62a00 x1860112241147008/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941449 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941433.697988:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941433.697996:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941433.698000:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941433.698036:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941433.698045:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941433.698047:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941433.698051:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941433.698058:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941433.698060:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941433.698064:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941433.698072:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62a00 x1860112241147008/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941449 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941433.698085:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62a00 x1860112241147008/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941449 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941433.698093:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941433.698095:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941433.698098:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941433.698102:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941433.698105:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941433.698108:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941433.698111:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941433.698118:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=12 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941433.698122:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941433.698125:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941433.698130:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941433.698134:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941433.698150:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941433.698153:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941433.698158:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941433.698163:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941433.698167:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941433.698170:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941433.698172:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941433.698175:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941433.698179:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d62a00 x1860112241147008/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941449 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941433.698187:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941433.698189:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941433.698191:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941433.698195:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941433.698198:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941433.698200:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941433.698202:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941433.698203:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941433.698206:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941433.698207:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941433.698209:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941433.698211:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941433.698214:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941433.698217:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941433.698220:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 202 00000400:00000200:1.0:1773941433.698225:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941433.698231:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941433.698236:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941433.698240:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941433.698243:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941433.698246:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941433.698249:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941433.698260:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941433.698262:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941433.698265:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941433.698268:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941433.698270:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941433.698274:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941433.698277:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941433.698279:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941433.698280:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941433.698284:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941433.698288:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941433.698292:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941433.698294:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941433.698298:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941433.698301:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941433.698304:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941433.698307:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941433.698312:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63100. 00000020:00000040:1.0:1773941433.698315:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941433.698318:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941433.698320:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941433.698323:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941433.698326:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941433.698328:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941433.698333:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734800. 00000100:00000001:1.0:1773941433.698338:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941433.698345:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63100 x1860112241147136/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941433.698354:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941433.698356:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941433.698361:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000025727364] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63100 x1860112241147136/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941433.698396:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941433.698398:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941433.698400:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941433.698403:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941433.698406:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941433.698408:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941433.698411:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6c00 going to sleep for 1 seconds 00000100:00000001:1.0:1773941433.698414:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941433.698418:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941433.698424:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62a00 x1860112241147008/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941449 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941433.698429:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941433.698436:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941433.698437:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941433.698442:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941433.698445:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941433.698449:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941433.698453:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941433.698458:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63100 x1860112241147136/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941433.698461:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62a00 x1860112241147008/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941449 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941433.698497:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941433.698502:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941433.698508:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241147008:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941433.698509:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941433.698516:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941433.698518:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941433.698521:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941433.698523:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941433.698529:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00100000:2.0:1773941433.698531:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241147136:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941433.698532:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941433.698541:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62a00 x1860112241147008/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941449 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941433.698546:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941433.698553:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941433.698556:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:1.0:1773941433.698556:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6c00. 02000000:00000001:2.0:1773941433.698565:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941433.698565:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941433.698569:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941433.698575:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941433.698575:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000010:2.0:1773941433.698584:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce7400. 02000000:00000001:2.0:1773941433.698588:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941433.698594:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941433.698597:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941433.698598:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000040:1.0:1773941433.698600:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=4 00000400:00000010:2.0:1773941433.698606:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af98f0. 00000020:00000001:1.0:1773941433.698606:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941433.698614:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000400:00000010:2.0:1773941433.698616:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34d48. 00010000:00010000:1.0:1773941433.698617:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c83 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941433.698624:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241147136, portal 25 00010000:00000001:1.0:1773941433.698626:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941433.698628:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941433.698631:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121082112 : -128249588469504 : ffff8b5b90d63100) 00010000:00010000:1.0:1773941433.698632:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c83 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941433.698640:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63100 x1860112241147136/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941443 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941433.698642:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941433.698649:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c83 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941433.698660:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941433.698674:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941433.698675:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941433.698681:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941433.698686:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c83 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941433.698687:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34550. 00000100:00000200:2.0:1773941433.698700:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241147136, offset 0 00010000:00000001:1.0:1773941433.698700:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941433.698738:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941433.698741:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941433.698744:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c83 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941433.698750:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f000 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941433.698753:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941433.698755:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c83 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941433.698767:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941433.698782:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941433.698782:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941433.698790:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941433.698793:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941433.698795:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=5 00000400:00000200:2.0:1773941433.698799:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000020:00000001:1.0:1773941433.698800:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941433.698806:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000400:00000200:2.0:1773941433.698807:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941433.698816:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:2.0:1773941433.698818:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00010000:1.0:1773941433.698820:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c83 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941433.698823:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941433.698835:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000001:1.0:1773941433.698839:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941433.698842:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c83 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941433.698854:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[22, 23] 10000000:01000000:1.0:1773941433.698859:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941433.698866:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941433.698868:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941433.698875:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:2.0:1773941433.698879:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941433.698879:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941433.698884:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941433.698886:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941433.698892:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941433.698895:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941433.698896:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [129:129:256:4294967295] 192.168.204.143@tcp LPNI seq info [24:24:8:4294967295] 00000020:00000040:1.0:1773941433.698898:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f000 with handle 0xa33a5be14eae1c83 from hash 00010000:00000001:1.0:1773941433.698902:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941433.698904:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941433.698906:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941433.698908:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941433.698911:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941433.698912:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941433.698921:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941433.698922:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941433.698923:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941433.698927:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941433.698934:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941433.698939:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c83 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941433.698944:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941433.698964:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941433.698965:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:2.0:1773941433.698969:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941433.698970:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941433.698976:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000800:00000010:2.0:1773941433.698979:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0600. 00010000:00000001:1.0:1773941433.698979:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941433.698986:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941433.698988:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941433.698990:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c83 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941433.698996:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000040:1.0:1773941433.699011:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ecd00 count: 0 00000800:00000200:2.0:1773941433.699012:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941433.699017:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0600 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941433.699044:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941433.699048:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941433.699049:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941433.699056:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941433.699057:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941433.699066:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294874062 00000100:00000001:2.0:1773941433.699074:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294874059 00000100:00000040:1.0:1773941433.699074:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62a00 x1860112241147008/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941449 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941433.699087:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294874062 00000100:00000001:2.0:1773941433.699098:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294874059 00000100:00000001:1.0:1773941433.699102:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941433.699104:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941433.699109:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941433.699110:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941433.699117:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735400. 02000000:00000001:1.0:1773941433.699123:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941433.699127:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941433.699129:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941433.699131:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000800:00000200:3.0:1773941433.699134:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941433.699139:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1773941433.699140:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000800:00000010:3.0:1773941433.699141:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0600. 02000000:00000010:1.0:1773941433.699147:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7400. 02000000:00000001:1.0:1773941433.699153:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:3.0:1773941433.699154:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941433.699155:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:1.0:1773941433.699156:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941433.699161:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000010:1.0:1773941433.699163:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62a00. 00000100:00000001:2.0:1773941433.699169:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1773941433.699175:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941433.699175:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941433.699175:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941433.699177:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941433.699182:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941433.699186:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941433.699188:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941433.699193:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941433.699199:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941433.699207:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0b00. 00000020:00000010:1.0:1773941433.699211:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7400. 00000020:00000010:1.0:1773941433.699217:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b7800. 00000040:00000010:1.0:1773941433.699221:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7a00. 00000020:00000010:1.0:1773941433.699226:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941433.699230:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941433.699235:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000400:00000200:3.0:1773941433.699238:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34550 00040000:00000010:1.0:1773941433.699241:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6e00. 00000400:00000010:3.0:1773941433.699245:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34550. 20000000:00000010:1.0:1773941433.699247:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b7000. 10000000:00000040:1.0:1773941433.699259:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000100:00000001:3.0:1773941433.699260:0:2347:0:(events.c:34:request_out_callback()) Process entered 10000000:00000001:1.0:1773941433.699268:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941433.699276:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b7800. 00000100:00000200:3.0:1773941433.699277:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63100 x1860112241147136/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941443 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000040:00000010:1.0:1773941433.699285:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7a00. 00000020:00000010:1.0:1773941433.699290:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941433.699299:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b6800. 00000100:00000001:3.0:1773941433.699307:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 80000000:00000010:1.0:1773941433.699307:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941433.699321:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6e00. 00000100:00000040:3.0:1773941433.699324:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63100 x1860112241147136/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941443 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 20000000:00000010:1.0:1773941433.699327:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941433.699332:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7400. 10000000:00000010:1.0:1773941433.699340:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0b00. 10000000:01000000:1.0:1773941433.699347:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941433.699356:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941433.699359:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 10000000:00020000:1.0:1773941433.699360:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000100:00000001:3.0:1773941433.699367:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941433.700578:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941433.700584:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941433.700586:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941433.700588:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941433.700591:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941433.700594:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941433.700599:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941433.700601:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941433.700604:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941433.700606:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321603d00 00000400:00000200:3.0:1773941433.700609:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941433.700612:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xba5 [1] + 0 00000800:00000001:3.0:1773941433.700621:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941433.700625:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941433.700627:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941433.700629:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941433.700632:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941433.700635:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63100 x1860112241147136/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941443 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941433.700727:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63100 x1860112241147136/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941443 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941433.700745:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941433.700755:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941433.700758:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941433.700784:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941433.700787:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941433.700791:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941433.700793:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941433.700795:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af98f0. 00000400:00000200:0.0:1773941433.700799:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34d48 00000400:00000010:0.0:1773941433.700800:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34d48. 00000100:00000001:0.0:1773941433.700803:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941433.700808:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63100 x1860112241147136/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941443 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941433.700815:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63100 x1860112241147136/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941443 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941433.700821:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941433.700822:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941433.700824:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941433.700826:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941433.700828:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941433.700829:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941433.700831:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941433.700837:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=12 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941433.700840:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941433.700842:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941433.700843:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941433.700845:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941433.700847:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941433.700849:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941433.700852:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63100 x1860112241147136/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941443 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941433.700857:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941433.700859:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941433.700861:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941433.700863:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941433.700875:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941433.700879:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941433.700881:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941433.700884:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941433.700886:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941433.700889:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941433.700892:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941433.700897:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941433 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941433.700900:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941433.700906:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941433.700909:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941433.700912:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941433.700915:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941433.700916:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941433.700920:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941433.700923:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941433.700925:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941433.700929:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941433.700933:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941433.700936:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941433.701144:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941433.701147:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941433.701152:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63100 x1860112241147136/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941443 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941433.701159:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241147136:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941433.701164:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941433.701165:0:12159:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:0.0:1773941433.701166:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941433.701170:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63100 x1860112241147136/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941443 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1773941433.701175:0:12159:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:0.0:1773941433.701180:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:2.0:1773941433.701181:0:12159:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941433.701183:0:12159:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941433.701185:0:12159:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:0.0:1773941433.701191:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:2.0:1773941433.701195:0:12159:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:0.0:1773941433.701197:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce7400. 00000001:00000001:2.0:1773941433.701198:0:12159:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941433.701205:0:12159:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941433.701205:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:2.0:1773941433.701207:0:12159:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:0.0:1773941433.701209:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:2.0:1773941433.701210:0:12159:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941433.701213:0:12159:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941433.701215:0:12159:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941433.701215:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941433.701219:0:12159:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:0.0:1773941433.701221:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:2.0:1773941433.701224:0:12159:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941433.701227:0:12159:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000010:0.0:1773941433.701227:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734800. 00000100:00000001:2.0:1773941433.701230:0:12159:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941433.701234:0:12159:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941433.701234:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:2.0:1773941433.701236:0:12159:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 02000000:00000001:0.0:1773941433.701239:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941433.701240:0:12159:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941433.701243:0:12159:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941433.701245:0:12159:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000010:0.0:1773941433.701245:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63100. 00000100:00000001:0.0:1773941433.701251:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941433.701253:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941433.701255:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941433.701256:0:12159:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941433.701258:0:12159:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941433.701259:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941433.701260:0:12159:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941433.701264:0:12159:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941433.701266:0:12159:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941433.701268:0:12159:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941433.701270:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941433.701271:0:12159:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941433.701276:0:12159:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941433.701277:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941433.701278:0:12159:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941433.701280:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941433.704519:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941433.704525:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941433.704527:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941433.704532:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:1.1:1773941433.710759:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ecd00. 00010000:00000010:1.1:1773941433.710773:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f000. 00000020:00000001:0.0:1773941434.207873:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941434.207882:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941434.207889:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941435.231898:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941435.231905:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941435.231911:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941435.551826:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941435.551835:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941435.551840:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941435.551843:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941435.551845:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941435.551846:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941435.551849:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941435.551874:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941435.551877:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941435.551881:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941435.551883:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941435.551894:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62a00. 00000020:00000040:1.0:1773941435.551901:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941435.551907:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941435.551911:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941435.551917:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941435.551926:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7400. 00000100:00000001:1.0:1773941435.551934:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941435.551939:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941435.551942:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941435.551950:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ecd00. 00010000:00000001:1.0:1773941435.551956:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941435.551959:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f000. 00000020:00000001:1.0:1773941435.551963:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941435.551966:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f000 with handle 0xa33a5be14eae1c8a to hash 00000020:00000001:1.0:1773941435.551967:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941435.551969:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941435.551972:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941435.551975:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941435.551978:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941435.551981:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c8a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941435.552014:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c8a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941435.552026:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c8a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941435.552031:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941435.552033:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941435.552047:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6c00. 00000100:00000001:1.0:1773941435.552049:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258984960 : -128249450566656 : ffff8b5b990e6c00) 00000100:00000001:1.0:1773941435.552051:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941435.552052:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121080320 : -128249588471296 : ffff8b5b90d62a00) 00000100:00000001:1.0:1773941435.552058:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941435.552062:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941435.552064:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941435.552066:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941435.552077:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62a00 x1860112241147264/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941435.552085:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941435.552087:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941435.552090:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941435.552094:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941435.552097:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941435.552101:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241147264:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941435.552107:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941435.552114:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941435.552117:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941435.552120:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941435.552125:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0736800. 02000000:00000001:1.0:1773941435.552128:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941435.552138:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941435.552143:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941435.552152:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941435.552163:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241147264, portal 25 00000100:00000001:1.0:1773941435.552165:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941435.552167:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121080320 : -128249588471296 : ffff8b5b90d62a00) 00000100:00000040:1.0:1773941435.552173:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62a00 x1860112241147264/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941451 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941435.552183:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941435.552186:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941435.552189:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1330. 00000100:00000200:1.0:1773941435.552199:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241147264, offset 0 00000400:00000200:1.0:1773941435.552208:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941435.552219:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941435.552227:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941435.552232:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941435.552236:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941435.552240:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941435.552242:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941435.552245:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[24, 23] 00000400:00000200:1.0:1773941435.552248:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941435.552251:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [130:130:256:4294967295] 192.168.204.153@tcp LPNI seq info [25:25:8:4294967295] 00000400:00000200:1.0:1773941435.552255:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941435.552264:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941435.552269:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941435.552273:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b800. 00000800:00000200:1.0:1773941435.552280:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941435.552287:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941435.552291:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b800 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941435.552314:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941435.552319:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941435.552323:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294875915 00000100:00000001:1.0:1773941435.552330:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294874062 00000100:00000001:1.0:1773941435.552336:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294875915 00000100:00000001:1.0:1773941435.552340:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294874062 00000100:00000001:1.0:1773941435.552344:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941435.552348:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941435.552351:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941435.552353:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941435.552358:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6c00 going to sleep for 11 seconds 00000100:00000001:1.0:1773941435.552362:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941435.552372:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941435.552481:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941435.552488:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b800. 00000400:00000200:3.0:1773941435.552496:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941435.552503:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941435.552511:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1330 00000400:00000010:3.0:1773941435.552515:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1330. 00000100:00000001:3.0:1773941435.552520:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941435.552526:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62a00 x1860112241147264/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941451 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941435.552539:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941435.552544:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d62a00 x1860112241147264/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941451 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941435.552552:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941435.552554:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941435.553612:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941435.553627:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941435.553631:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941435.553643:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941435.553648:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941435.553652:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941435.553657:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941435.553660:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941435.553664:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941435.553668:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321603d80 00000400:00000200:3.0:1773941435.553671:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941435.553675:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xbb5 [1] + 224 00000800:00000001:3.0:1773941435.553681:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941435.553686:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941435.553688:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941435.553740:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941435.553745:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941435.553749:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62a00 x1860112241147264/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941451 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941435.553757:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d62a00 x1860112241147264/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941451 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941435.553770:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941435.553777:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941435.553780:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941435.553803:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941435.553810:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941435.553811:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941435.553813:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941435.553816:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941435.553818:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941435.553820:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941435.553824:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62a00 x1860112241147264/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941451 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941435.553832:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62a00 x1860112241147264/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941451 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941435.553837:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941435.553839:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941435.553841:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941435.553843:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941435.553846:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941435.553848:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941435.553850:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941435.553855:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=14 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941435.553859:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941435.553863:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941435.553868:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941435.553871:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941435.553875:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941435.553878:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941435.553883:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941435.553889:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941435.553893:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941435.553896:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941435.553898:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941435.553901:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941435.553905:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d62a00 x1860112241147264/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941451 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941435.553917:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941435.553918:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941435.553920:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941435.553922:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941435.553925:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941435.553927:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941435.553928:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941435.553929:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941435.553930:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941435.553931:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941435.553932:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941435.553934:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941435.553936:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941435.553938:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941435.553941:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 204 00000400:00000200:1.0:1773941435.553944:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941435.553948:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941435.553951:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941435.553954:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941435.553956:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941435.553958:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941435.553960:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941435.553962:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941435.553964:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941435.553966:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941435.553967:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941435.553969:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941435.553970:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941435.553972:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941435.553973:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941435.553975:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941435.553977:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941435.553983:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941435.553985:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941435.553999:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941435.554003:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941435.554005:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941435.554007:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941435.554009:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941435.554013:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62d80. 00000020:00000040:1.0:1773941435.554015:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941435.554019:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941435.554021:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941435.554023:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941435.554024:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941435.554026:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941435.554029:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734000. 00000100:00000001:1.0:1773941435.554032:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941435.554037:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d62d80 x1860112241147392/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941435.554043:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941435.554045:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941435.554048:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002abf64a5] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d62d80 x1860112241147392/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941435.554066:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941435.554068:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941435.554069:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941435.554072:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941435.554074:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941435.554075:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941435.554077:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6c00 going to sleep for 1 seconds 00000100:00000001:1.0:1773941435.554078:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941435.554080:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941435.554083:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62a00 x1860112241147264/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941451 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941435.554097:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941435.554101:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941435.554103:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941435.554105:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941435.554110:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62a00 x1860112241147264/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941451 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941435.554112:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941435.554121:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941435.554128:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:1.0:1773941435.554133:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241147264:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000040:0.0:1773941435.554144:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62d80 x1860112241147392/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941435.554148:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941435.554159:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941435.554167:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941435.554168:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941435.554170:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941435.554173:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1773941435.554175:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941435.554180:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1773941435.554184:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62a00 x1860112241147264/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941451 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1773941435.554184:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941435.554201:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241147392:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941435.554218:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941435.554219:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:1.0:1773941435.554227:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6c00. 02000000:00000001:0.0:1773941435.554227:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941435.554232:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941435.554234:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941435.554241:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:0.0:1773941435.554242:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941435.554250:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941435.554253:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000010:0.0:1773941435.554254:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0c00. 00000020:00000001:1.0:1773941435.554255:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941435.554257:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=4 00000020:00000001:1.0:1773941435.554259:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941435.554262:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00010000:1.0:1773941435.554265:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c8a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941435.554276:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941435.554290:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:0.0:1773941435.554295:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941435.554296:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c8a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941435.554300:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941435.554307:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00010000:00000001:1.0:1773941435.554311:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941435.554315:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c8a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:0.0:1773941435.554315:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241147392, portal 25 00000100:00000001:0.0:1773941435.554321:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941435.554325:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121081216 : -128249588470400 : ffff8b5b90d62d80) 00010000:00000001:1.0:1773941435.554329:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941435.554333:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941435.554336:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c8a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941435.554337:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62d80 x1860112241147392/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941445 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941435.554375:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:0.0:1773941435.554375:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941435.554383:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941435.554384:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941435.554387:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c000. 00010000:00000001:1.0:1773941435.554393:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000200:0.0:1773941435.554393:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241147392, offset 0 00010000:00010000:1.0:1773941435.554401:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c8a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941435.554406:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941435.554415:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f000 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:0.0:1773941435.554416:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:1.0:1773941435.554426:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941435.554429:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c8a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941435.554434:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:1.0:1773941435.554450:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:0.0:1773941435.554451:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941435.554455:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941435.554458:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1773941435.554461:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000020:00000040:1.0:1773941435.554463:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=5 00000020:00000001:1.0:1773941435.554467:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000400:00000200:0.0:1773941435.554468:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941435.554477:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000400:00000200:0.0:1773941435.554481:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941435.554487:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:0.0:1773941435.554494:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[24, 25] 00010000:00010000:1.0:1773941435.554495:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c8a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941435.554515:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 10000000:00000001:1.0:1773941435.554520:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941435.554531:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [131:131:256:4294967295] 192.168.204.143@tcp LPNI seq info [26:26:8:4294967295] 10000000:00010000:1.0:1773941435.554532:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c8a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941435.554554:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:01000000:1.0:1773941435.554558:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941435.554561:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941435.554563:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941435.554569:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941435.554576:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:0.0:1773941435.554579:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941435.554582:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941435.554585:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941435.554592:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000800:00000200:0.0:1773941435.554593:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941435.554599:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941435.554603:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f000 with handle 0xa33a5be14eae1c8a from hash 00000800:00000010:0.0:1773941435.554603:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00010000:00000001:1.0:1773941435.554609:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941435.554610:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941435.554611:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941435.554612:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941435.554613:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941435.554619:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941435.554621:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941435.554624:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:0.0:1773941435.554625:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941435.554627:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941435.554633:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941435.554637:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c8a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941435.554645:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941435.554657:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:0.0:1773941435.554658:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941435.554660:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941435.554663:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941435.554666:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941435.554667:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941435.554669:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941435.554669:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941435.554671:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c8a lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941435.554675:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1773941435.554678:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ecd00 count: 0 00000400:00000200:0.0:1773941435.554689:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941435.554693:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294875917 00010000:00000001:1.0:1773941435.554736:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941435.554738:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941435.554741:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294875915 00000100:00000001:1.0:1773941435.554742:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941435.554751:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62a00 x1860112241147264/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941451 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941435.554756:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294875918 00000100:00000001:1.0:1773941435.554769:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941435.554770:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294875915 02000000:00000001:1.0:1773941435.554772:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941435.554777:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0736800. 00000100:00000001:0.0:1773941435.554778:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.0:1773941435.554784:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1773941435.554784:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941435.554785:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941435.554790:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941435.554799:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000800:00000010:3.0:1773941435.554801:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000100:00000001:0.0:1773941435.554801:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941435.554817:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941435.554817:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941435.554818:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:1.0:1773941435.554825:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7400. 02000000:00000001:1.0:1773941435.554836:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941435.554840:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941435.554841:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941435.554847:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000010:1.0:1773941435.554848:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62a00. 00000100:00000001:0.0:1773941435.554859:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941435.554861:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941435.554862:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941435.554864:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941435.554867:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941435.554868:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941435.554876:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941435.554883:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941435.554888:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000400:00000200:3.0:1773941435.554894:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 10000000:00000010:1.0:1773941435.554896:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0b00. 00000400:00000200:3.0:1773941435.554899:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c000 00000400:00000010:3.0:1773941435.554902:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c000. 00000020:00000010:1.0:1773941435.554903:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7400. 00000100:00000001:3.0:1773941435.554906:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941435.554911:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62d80 x1860112241147392/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941445 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941435.554912:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b7000. 00000040:00000010:1.0:1773941435.554919:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6e00. 00000100:00000001:3.0:1773941435.554921:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941435.554926:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62d80 x1860112241147392/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941445 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941435.554926:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941435.554931:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941435.554938:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000001:3.0:1773941435.554943:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:1.0:1773941435.554945:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7a00. 00000100:00000001:3.0:1773941435.554946:0:2347:0:(events.c:68:request_out_callback()) Process leaving 20000000:00000010:1.0:1773941435.554952:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b7800. 10000000:00000040:1.0:1773941435.554957:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941435.554959:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941435.554964:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b7000. 00000040:00000010:1.0:1773941435.554968:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941435.554976:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941435.554980:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941435.554982:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941435.555001:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7a00. 20000000:00000010:1.0:1773941435.555004:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941435.555006:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7400. 10000000:00000010:1.0:1773941435.555009:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0b00. 10000000:01000000:1.0:1773941435.555011:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941435.555014:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941435.555016:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941435.555017:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941435.555018:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941435.555020:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941435.555022:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941435.556097:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941435.556103:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941435.556105:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941435.556107:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941435.556111:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941435.556114:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941435.556118:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941435.556121:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941435.556124:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941435.556178:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321603e00 00000400:00000200:3.0:1773941435.556182:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941435.556185:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xbbd [1] + 0 00000800:00000001:3.0:1773941435.556189:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941435.556195:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941435.556197:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941435.556199:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941435.556202:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941435.556206:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62d80 x1860112241147392/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941445 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941435.556214:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d62d80 x1860112241147392/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941445 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941435.556229:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941435.556233:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941435.556236:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941435.556261:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941435.556263:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941435.556268:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941435.556269:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941435.556272:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941435.556276:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941435.556278:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941435.556281:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941435.556286:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62d80 x1860112241147392/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941445 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941435.556294:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62d80 x1860112241147392/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941445 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941435.556299:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941435.556300:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941435.556302:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941435.556304:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941435.556306:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941435.556307:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941435.556309:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941435.556315:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=14 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941435.556326:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941435.556329:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941435.556331:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941435.556334:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941435.556336:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941435.556340:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941435.556345:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62d80 x1860112241147392/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941445 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941435.556353:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941435.556356:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941435.556358:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941435.556361:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941435.556368:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941435.556371:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941435.556373:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941435.556375:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941435.556377:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941435.556379:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941435.556381:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941435.556384:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941435 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941435.556386:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941435.556392:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941435.556393:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941435.556396:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941435.556397:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941435.556399:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941435.556402:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941435.556403:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941435.556405:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941435.556408:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941435.556411:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941435.556413:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941435.556665:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941435.556670:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941435.556678:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62d80 x1860112241147392/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941445 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941435.556687:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241147392:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941435.556692:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941435.556695:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941435.556698:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62d80 x1860112241147392/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941445 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941435.556731:0:12160:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941435.556740:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941435.556744:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941435.556748:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0c00. 02000000:00000001:2.0:1773941435.556753:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941435.556757:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00080000:0.0:1773941435.556758:0:12160:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000020:00000040:2.0:1773941435.556760:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941435.556768:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00100000:0.0:1773941435.556769:0:12160:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000010:2.0:1773941435.556771:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734000. 02000000:00000001:2.0:1773941435.556776:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1773941435.556776:0:12160:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941435.556782:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941435.556785:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62d80. 10000000:01000000:0.0:1773941435.556786:0:12160:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941435.556789:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941435.556791:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:0.0:1773941435.556793:0:12160:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941435.556794:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:0.0:1773941435.556800:0:12160:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941435.556801:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941435.556806:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941435.556808:0:12160:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941435.556809:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941435.556813:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941435.556820:0:12160:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941435.556823:0:12160:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941435.556826:0:12160:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941435.556831:0:12160:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941435.556833:0:12160:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941435.556839:0:12160:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941435.556842:0:12160:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941435.556845:0:12160:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941435.556847:0:12160:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941435.556850:0:12160:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941435.556852:0:12160:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941435.556854:0:12160:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941435.556856:0:12160:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941435.556880:0:12160:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941435.556883:0:12160:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941435.556885:0:12160:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941435.556891:0:12160:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941435.556899:0:12160:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941435.556903:0:12160:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941435.556907:0:12160:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941435.556910:0:12160:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941435.556912:0:12160:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941435.559748:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ecd00. 00010000:00000010:1.1:1773941435.563739:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f000. 00000020:00000001:0.0:1773941436.255861:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941436.255871:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941436.255877:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941437.279914:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941437.279920:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941437.279923:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1773941437.343880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1773941437.343891:0:37:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1773941437.343902:0:37:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021ca80. 00000020:00000040:2.0:1773941437.343914:0:37:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941437.343920:0:37:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941437.343924:0:37:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941437.343930:0:37:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941437.343941:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b98bb0800. 00000100:00000001:2.0:1773941437.343951:0:37:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941437.343979:0:37:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS req@ffff8b5b9021ca80 x1860112241147520/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941437.343995:0:37:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941437.344001:0:37:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941437.344011:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000005f3b654] to pc [ptlrpcd_01_00+0] req@ffff8b5b9021ca80 x1860112241147520/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941437.344075:0:37:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941437.344081:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (212) 00000100:00000001:2.0:1773941437.344097:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941437.344101:0:2354:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941437.344104:0:2354:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941437.344111:0:2354:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021ca80 x1860112241147520/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941437.344116:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941437.344118:0:2354:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941437.344124:0:2354:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941437.344125:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941437.344143:0:2354:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941437.344147:0:2354:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941437.344151:0:2354:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941437.344154:0:2354:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:1080e6b5-6a6c-4762-b0ec-c785990fec17:2354:1860112241147520:192.168.204.143@tcp:400:kworker.0 00000100:00000001:2.0:1773941437.344158:0:2354:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941437.344165:0:2354:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941437.344167:0:2354:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941437.344171:0:2354:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941437.344175:0:2354:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b90e7e400. 02000000:00000001:2.0:1773941437.344177:0:2354:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941437.344187:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941437.344191:0:2354:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941437.344199:0:2354:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000200:2.0:1773941437.344209:0:2354:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241147520, portal 25 00000100:00000001:2.0:1773941437.344212:0:2354:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941437.344215:0:2354:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109259392 : -128249600292224 : ffff8b5b9021ca80) 00000100:00000040:2.0:1773941437.344222:0:2354:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021ca80 x1860112241147520/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941453 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941437.344228:0:2354:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941437.344231:0:2354:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941437.344233:0:2354:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34220. 00000100:00000200:2.0:1773941437.344235:0:2354:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241147520, offset 0 00000400:00000200:2.0:1773941437.344241:0:2354:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941437.344256:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941437.344264:0:2354:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941437.344270:0:2354:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941437.344275:0:2354:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941437.344280:0:2354:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941437.344284:0:2354:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941437.344288:0:2354:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[26, 25] 00000400:00000200:2.0:1773941437.344292:0:2354:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:2.0:1773941437.344297:0:2354:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [132:132:256:4294967295] 192.168.204.153@tcp LPNI seq info [27:27:8:4294967295] 00000400:00000200:2.0:1773941437.344303:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941437.344314:0:2354:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:2.0:1773941437.344321:0:2354:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:2.0:1773941437.344327:0:2354:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0e00. 00000800:00000200:2.0:1773941437.344337:0:2354:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:2.0:1773941437.344347:0:2354:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:2.0:1773941437.344353:0:2354:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0e00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1773941437.344368:0:2354:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941437.344371:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941437.344374:0:2354:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294877707 00000100:00000001:2.0:1773941437.344386:0:2354:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294875917 00000100:00000001:2.0:1773941437.344389:0:2354:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294877707 00000100:00000001:2.0:1773941437.344396:0:2354:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294875918 00000100:00000001:2.0:1773941437.344398:0:2354:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941437.344400:0:2354:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.344403:0:2354:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.344405:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941437.344414:0:2354:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941437.344416:0:2354:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1773941437.344419:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941437.344420:0:2354:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941437.344422:0:2354:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.344432:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941437.344436:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941437.344447:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0e00. 00000400:00000200:3.0:1773941437.344453:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941437.344460:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941437.344466:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34220 00000400:00000010:3.0:1773941437.344469:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34220. 00000100:00000001:3.0:1773941437.344473:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941437.344480:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021ca80 x1860112241147520/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941453 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941437.344489:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941437.344492:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021ca80 x1860112241147520/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941453 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941437.344497:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941437.344499:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941437.345495:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.345508:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941437.345510:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.345513:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941437.345521:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941437.345527:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941437.345535:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941437.345579:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941437.345585:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941437.345590:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321603e80 00000400:00000200:3.0:1773941437.345595:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941437.345601:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xbc5 [1] + 224 00000800:00000001:3.0:1773941437.345610:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.345618:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941437.345620:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941437.345625:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941437.345630:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941437.345636:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021ca80 x1860112241147520/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941453 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941437.345646:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b9021ca80 x1860112241147520/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941453 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941437.345672:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941437.345679:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.345683:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941437.345695:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941437.345698:0:2354:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941437.345728:0:2354:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941437.345730:0:2354:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941437.345734:0:2354:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941437.345738:0:2354:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941437.345741:0:2354:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941437.345744:0:2354:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941437.345751:0:2354:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021ca80 x1860112241147520/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941453 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941437.345760:0:2354:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021ca80 x1860112241147520/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941453 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941437.345765:0:2354:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941437.345766:0:2354:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941437.345768:0:2354:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941437.345771:0:2354:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941437.345773:0:2354:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941437.345775:0:2354:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941437.345778:0:2354:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941437.345784:0:2354:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=16 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941437.345787:0:2354:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941437.345789:0:2354:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941437.345791:0:2354:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941437.345794:0:2354:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941437.345796:0:2354:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941437.345800:0:2354:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941437.345804:0:2354:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941437.345807:0:2354:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941437.345810:0:2354:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941437.345812:0:2354:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941437.345814:0:2354:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941437.345819:0:2354:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b9021ca80 x1860112241147520/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941453 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941437.345835:0:2354:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941437.345837:0:2354:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941437.345839:0:2354:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941437.345844:0:2354:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941437.345847:0:2354:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941437.345849:0:2354:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.345852:0:2354:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941437.345853:0:2354:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941437.345854:0:2354:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941437.345856:0:2354:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.345858:0:2354:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941437.345861:0:2354:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941437.345863:0:2354:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941437.345867:0:2354:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941437.345871:0:2354:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 206 00000400:00000200:2.0:1773941437.345876:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941437.345880:0:2354:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941437.345883:0:2354:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941437.345887:0:2354:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941437.345889:0:2354:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941437.345891:0:2354:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:2.0:1773941437.345893:0:2354:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941437.345895:0:2354:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941437.345898:0:2354:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941437.345900:0:2354:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941437.345902:0:2354:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941437.345903:0:2354:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:2.0:1773941437.345905:0:2354:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941437.345907:0:2354:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941437.345908:0:2354:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941437.345910:0:2354:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:2.0:1773941437.345913:0:2354:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:2.0:1773941437.345920:0:2354:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:2.0:1773941437.345922:0:2354:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941437.345924:0:2354:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941437.345927:0:2354:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941437.345929:0:2354:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.345931:0:2354:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941437.345934:0:2354:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941437.345937:0:2354:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021d500. 00000020:00000040:2.0:1773941437.345939:0:2354:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:2.0:1773941437.345942:0:2354:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941437.345944:0:2354:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.345946:0:2354:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941437.345948:0:2354:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941437.345950:0:2354:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941437.345967:0:2354:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0ce4000. 00000100:00000001:2.0:1773941437.345971:0:2354:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941437.345978:0:2354:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b9021d500 x1860112241147648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941437.345986:0:2354:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941437.345989:0:2354:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941437.345992:0:2354:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1896fa] to pc [ptlrpcd_rcv+-1] req@ffff8b5b9021d500 x1860112241147648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941437.346018:0:2354:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.346021:0:2354:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941437.346023:0:2354:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:2.0:1773941437.346029:0:2354:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021ca80 x1860112241147520/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941453 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941437.346038:0:2354:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941437.346042:0:2354:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941437.346045:0:2354:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941437.346050:0:2354:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021ca80 x1860112241147520/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941453 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941437.346060:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941437.346064:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941437.346067:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1773941437.346077:0:2354:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:1080e6b5-6a6c-4762-b0ec-c785990fec17:2354:1860112241147520:192.168.204.143@tcp:400:kworker.0 00000100:00000040:0.0:1773941437.346078:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021d500 x1860112241147648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941437.346082:0:2354:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941437.346084:0:2354:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941437.346088:0:2354:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021ca80 x1860112241147520/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941453 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941437.346117:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941437.346118:0:2354:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941437.346120:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941437.346124:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941437.346125:0:2354:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1773941437.346126:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000010:2.0:1773941437.346129:0:2354:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b90e7e400. 02000000:00000001:0.0:1773941437.346130:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941437.346133:0:2354:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:0.0:1773941437.346137:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241147648:192.168.204.143@tcp:250:ptlrpcd_01_00.0 00000020:00000001:2.0:1773941437.346138:0:2354:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941437.346141:0:2354:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941437.346146:0:2354:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941437.346146:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941437.346151:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:2.0:1773941437.346152:0:2354:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b98bb0800. 02000000:00000001:0.0:1773941437.346154:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941437.346157:0:2354:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941437.346158:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1773941437.346161:0:2354:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 02000000:00000010:0.0:1773941437.346164:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c3000. 00000100:00000010:2.0:1773941437.346165:0:2354:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021ca80. 02000000:00000001:0.0:1773941437.346169:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.346170:0:2354:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941437.346174:0:2354:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941437.346177:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941437.346178:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1773941437.346181:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000100:00000001:2.0:1773941437.346184:0:2354:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000400:00000010:0.0:1773941437.346186:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000001:2.0:1773941437.346187:0:2354:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.346192:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:0.0:1773941437.346194:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241147648, portal 25 00000100:00000001:2.0:1773941437.346195:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941437.346199:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941437.346202:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109262080 : -128249600289536 : ffff8b5b9021d500) 00000100:00000040:0.0:1773941437.346206:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021d500 x1860112241147648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941447 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941437.346212:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941437.346214:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941437.346215:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c2a8. 00000100:00000200:0.0:1773941437.346218:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241147648, offset 0 00000400:00000200:0.0:1773941437.346222:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941437.346226:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941437.346229:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941437.346231:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941437.346233:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941437.346235:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941437.346236:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941437.346242:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[26, 27] 00000400:00000200:0.0:1773941437.346246:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941437.346248:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [133:133:256:4294967295] 192.168.204.143@tcp LPNI seq info [28:28:8:4294967295] 00000400:00000200:0.0:1773941437.346252:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941437.346258:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941437.346261:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941437.346264:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00000800:00000200:0.0:1773941437.346269:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941437.346276:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941437.346278:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:0.0:1773941437.346292:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941437.346294:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941437.346296:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294877709 00000100:00000001:0.0:1773941437.346299:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294877707 00000100:00000001:0.0:1773941437.346302:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294877709 00000100:00000001:0.0:1773941437.346305:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294877707 00000100:00000001:0.0:1773941437.346307:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941437.346309:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941437.346310:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941437.346312:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941437.346315:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941437.346316:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1773941437.346318:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941437.346319:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941437.346320:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941437.346322:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941437.346404:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941437.346408:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000400:00000200:3.0:1773941437.346413:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941437.346417:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941437.346421:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c2a8 00000400:00000010:3.0:1773941437.346422:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c2a8. 00000100:00000001:3.0:1773941437.346425:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941437.346429:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021d500 x1860112241147648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941447 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941437.346436:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941437.346439:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021d500 x1860112241147648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941447 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941437.346444:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941437.346445:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941437.348223:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.348231:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941437.348234:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.348235:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941437.348240:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941437.348244:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941437.348249:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941437.348252:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941437.348255:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941437.348258:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321603f00 00000400:00000200:3.0:1773941437.348261:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941437.348265:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xbd5 [1] + 0 00000800:00000001:3.0:1773941437.348270:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.348336:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941437.348339:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941437.348352:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941437.348357:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941437.348363:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021d500 x1860112241147648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941447 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941437.348375:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b9021d500 x1860112241147648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941447 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941437.348406:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941437.348413:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.348417:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941437.348445:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941437.348448:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941437.348454:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941437.348455:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941437.348458:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941437.348461:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941437.348468:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941437.348470:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941437.348475:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021d500 x1860112241147648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941447 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941437.348482:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021d500 x1860112241147648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941447 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941437.348487:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941437.348489:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941437.348490:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941437.348492:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941437.348493:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941437.348495:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941437.348497:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941437.348500:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=16 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941437.348503:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941437.348506:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941437.348507:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941437.348511:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941437.348514:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941437.348517:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941437.348522:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021d500 x1860112241147648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941447 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941437.348530:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941437.348532:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941437.348534:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941437.348537:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941437.348544:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941437.348546:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941437.348548:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941437.348550:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941437.348553:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941437.348556:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941437.348559:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941437.348564:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941437 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941437.348574:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941437.348583:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941437.348585:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941437.348587:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941437.348589:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941437.348591:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941437.348596:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941437.348599:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941437.348601:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941437.348605:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941437.348609:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941437.348612:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941437.348886:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.348891:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941437.348900:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021d500 x1860112241147648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941447 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941437.348910:0:12162:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941437.348912:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241147648:192.168.204.143@tcp:250:ptlrpcd_01_00.0 00000100:00000001:2.0:1773941437.348920:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941437.348925:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1773941437.348925:0:12162:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941437.348933:0:12162:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:2.0:1773941437.348934:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021d500 x1860112241147648/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941447 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941437.348938:0:12162:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941437.348940:0:12162:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941437.348946:0:12162:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941437.348951:0:12162:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941437.348967:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941437.348979:0:12162:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941437.348983:0:12162:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941437.348985:0:12162:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941437.348987:0:12162:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941437.348989:0:12162:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941437.348992:0:12162:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941437.348996:0:12162:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941437.348999:0:12162:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941437.349002:0:12162:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941437.349005:0:12162:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941437.349008:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:0.0:1773941437.349009:0:12162:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941437.349014:0:12162:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:2.0:1773941437.349016:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c3000. 10000000:01000000:0.0:1773941437.349017:0:12162:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941437.349022:0:12162:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 02000000:00000001:2.0:1773941437.349023:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941437.349026:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941437.349027:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941437.349029:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941437.349031:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0ce4000. 02000000:00000001:2.0:1773941437.349034:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941437.349035:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941437.349037:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021d500. 00000100:00000001:2.0:1773941437.349039:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941437.349040:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941437.349045:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941437.349046:0:12162:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:2.0:1773941437.349050:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000001:0.0:1773941437.349050:0:12162:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.349053:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.349055:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941437.349056:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941437.349063:0:12162:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941437.349070:0:12162:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941437.349078:0:12162:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941437.349082:0:12162:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941437.349085:0:12162:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 10000000:00000001:1.0:1773941437.349089:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 00000020:00000001:0.0:1773941437.349089:0:12162:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000040:1.0:1773941437.349100:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 00000100:00000001:0.0:1773941437.349101:0:12162:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941437.349109:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941437.349113:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941437.349114:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941437.349115:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941437.349118:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941437.349120:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941437.349122:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941437.349125:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941437.349127:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941437.349135:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62a00. 00000020:00000040:1.0:1773941437.349140:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941437.349144:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941437.349148:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941437.349155:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941437.349163:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7400. 00000100:00000001:1.0:1773941437.349173:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941437.349178:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941437.349181:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941437.349190:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ecd00. 00010000:00000001:1.0:1773941437.349197:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941437.349208:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f000. 00000020:00000001:1.0:1773941437.349211:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941437.349213:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f000 with handle 0xa33a5be14eae1c91 to hash 00000020:00000001:1.0:1773941437.349215:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941437.349216:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941437.349218:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941437.349222:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941437.349224:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941437.349228:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c91 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941437.349236:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c91 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941437.349245:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c91 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941437.349251:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941437.349254:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941437.349260:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941437.349262:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941437.349265:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941437.349266:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121080320 : -128249588471296 : ffff8b5b90d62a00) 00000100:00000001:1.0:1773941437.349271:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941437.349275:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941437.349278:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941437.349279:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941437.349288:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62a00 x1860112241147776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941437.349298:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941437.349301:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941437.349305:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941437.349308:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941437.349316:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941437.349321:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241147776:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941437.349324:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941437.349328:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941437.349330:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941437.349332:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941437.349335:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0737400. 02000000:00000001:1.0:1773941437.349337:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941437.349342:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941437.349345:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941437.349349:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941437.349354:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241147776, portal 25 00000100:00000001:1.0:1773941437.349356:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941437.349357:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121080320 : -128249588471296 : ffff8b5b90d62a00) 00000100:00000040:1.0:1773941437.349361:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62a00 x1860112241147776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941453 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941437.349366:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941437.349368:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941437.349370:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f12a8. 00000100:00000200:1.0:1773941437.349372:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241147776, offset 0 00000400:00000200:1.0:1773941437.349377:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941437.349383:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941437.349388:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941437.349392:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941437.349395:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941437.349398:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941437.349400:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941437.349403:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[28, 27] 00000400:00000200:1.0:1773941437.349405:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941437.349408:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [134:134:256:4294967295] 192.168.204.153@tcp LPNI seq info [29:29:8:4294967295] 00000400:00000200:1.0:1773941437.349411:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941437.349417:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941437.349425:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941437.349429:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b100. 00000800:00000200:1.0:1773941437.349434:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941437.349439:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941437.349443:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b100 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941437.349465:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941437.349469:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941437.349471:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294877712 00000100:00000001:1.0:1773941437.349476:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294877709 00000100:00000001:1.0:1773941437.349484:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294877712 00000100:00000001:1.0:1773941437.349489:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294877709 00000100:00000001:1.0:1773941437.349497:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941437.349501:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941437.349503:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941437.349504:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941437.349506:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941437.349508:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941437.349511:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941437.349547:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941437.349552:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b100. 00000400:00000200:3.0:1773941437.349558:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941437.349565:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941437.349569:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f12a8 00000400:00000010:3.0:1773941437.349572:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f12a8. 00000100:00000001:3.0:1773941437.349575:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941437.349581:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62a00 x1860112241147776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941453 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941437.349591:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941437.349595:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d62a00 x1860112241147776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941453 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941437.349603:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941437.349605:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941437.350473:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.350480:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941437.350483:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.350485:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941437.350491:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941437.350496:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941437.350502:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941437.350505:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941437.350509:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941437.350518:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321603f80 00000400:00000200:3.0:1773941437.350521:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941437.350525:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xbdd [1] + 224 00000800:00000001:3.0:1773941437.350529:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.350534:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941437.350535:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941437.350538:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941437.350541:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941437.350544:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62a00 x1860112241147776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941453 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941437.350551:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d62a00 x1860112241147776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941453 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941437.350568:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941437.350574:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.350576:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941437.350600:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941437.350607:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941437.350609:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941437.350612:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941437.350618:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941437.350621:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941437.350624:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941437.350631:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62a00 x1860112241147776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941453 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941437.350641:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62a00 x1860112241147776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941453 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941437.350649:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941437.350651:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941437.350661:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941437.350663:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941437.350666:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941437.350668:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941437.350671:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941437.350676:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=16 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941437.350679:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941437.350683:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941437.350687:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941437.350690:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941437.350694:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941437.350697:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941437.350774:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941437.350780:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941437.350784:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941437.350786:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941437.350788:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941437.350790:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941437.350794:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d62a00 x1860112241147776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941453 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941437.350803:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941437.350804:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941437.350806:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941437.350819:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941437.350826:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941437.350827:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941437.350830:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941437.350831:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941437.350832:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941437.350834:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941437.350835:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941437.350838:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941437.350840:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941437.350843:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941437.350854:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 207 00000400:00000200:1.0:1773941437.350860:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941437.350865:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941437.350869:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941437.350872:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941437.350875:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941437.350877:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941437.350880:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941437.350882:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941437.350884:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941437.350887:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941437.350889:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941437.350892:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941437.350895:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941437.350897:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941437.350899:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941437.350901:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941437.350904:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941437.350908:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941437.350911:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941437.350913:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941437.350916:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941437.350919:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941437.350922:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941437.350924:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941437.350928:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62680. 00000020:00000040:1.0:1773941437.350931:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941437.350934:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941437.350936:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941437.350938:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941437.350940:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941437.350942:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941437.350965:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0736800. 00000100:00000001:1.0:1773941437.350970:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941437.350977:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d62680 x1860112241147904/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941437.350985:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941437.350988:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941437.350992:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048151604] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d62680 x1860112241147904/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941437.351015:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941437.351019:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941437.351021:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941437.351023:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941437.351026:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941437.351027:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941437.351029:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941437.351032:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941437.351034:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941437.351039:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62a00 x1860112241147776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941453 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941437.351046:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941437.351048:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941437.351051:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941437.351053:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941437.351055:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941437.351058:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941437.351061:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941437.351066:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62680 x1860112241147904/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941437.351070:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62a00 x1860112241147776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941453 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941437.351090:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941437.351093:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941437.351095:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241147776:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941437.351098:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941437.351102:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941437.351104:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941437.351105:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941437.351112:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:2.0:1773941437.351113:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241147904:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941437.351118:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941437.351120:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941437.351122:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941437.351125:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941437.351127:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941437.351129:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941437.351135:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce4000. 00000100:00000040:1.0:1773941437.351137:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62a00 x1860112241147776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941453 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941437.351147:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941437.351149:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941437.351154:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000400:00000200:2.0:1773941437.351155:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941437.351160:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000400:00000010:2.0:1773941437.351162:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000100:00000001:1.0:1773941437.351164:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000010:2.0:1773941437.351169:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00000001:1.0:1773941437.351169:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941437.351174:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000200:2.0:1773941437.351176:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241147904, portal 25 00000020:00000001:1.0:1773941437.351178:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941437.351181:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000020:00000040:1.0:1773941437.351182:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=4 00000100:00000001:2.0:1773941437.351184:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121079424 : -128249588472192 : ffff8b5b90d62680) 00000020:00000001:1.0:1773941437.351188:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000100:00000040:2.0:1773941437.351193:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62680 x1860112241147904/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941447 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941437.351194:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00010000:1.0:1773941437.351203:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c91 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941437.351204:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941437.351208:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941437.351210:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34198. 00000100:00000200:2.0:1773941437.351214:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241147904, offset 0 00000400:00000200:2.0:1773941437.351220:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941437.351221:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:2.0:1773941437.351226:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941437.351231:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00010000:1.0:1773941437.351231:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c91 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941437.351237:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941437.351242:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941437.351245:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941437.351249:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941437.351249:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:2.0:1773941437.351255:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[28, 29] 00010000:00010000:1.0:1773941437.351255:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c91 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941437.351262:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941437.351266:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [135:135:256:4294967295] 192.168.204.143@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:2.0:1773941437.351275:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941437.351276:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941437.351280:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941437.351284:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00010000:1.0:1773941437.351285:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c91 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941437.351291:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941437.351296:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0800. 00010000:00000001:1.0:1773941437.351301:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000800:00000200:2.0:1773941437.351302:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941437.351305:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941437.351310:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941437.351312:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000800:00000200:2.0:1773941437.351314:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0800 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:1.0:1773941437.351318:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c91 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941437.351327:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941437.351334:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f000 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:2.0:1773941437.351339:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:1.0:1773941437.351340:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:2.0:1773941437.351342:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294877714 10000000:00010000:1.0:1773941437.351346:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c91 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941437.351348:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294877712 00000100:00000001:2.0:1773941437.351354:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294877714 00010000:00000001:1.0:1773941437.351360:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:2.0:1773941437.351362:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294877712 00010000:00000001:1.0:1773941437.351364:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:2.0:1773941437.351366:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1773941437.351368:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941437.351369:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.351373:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941437.351373:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=5 00000100:00000001:2.0:1773941437.351378:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1773941437.351381:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000100:00000001:2.0:1773941437.351384:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941437.351386:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000100:00000001:2.0:1773941437.351387:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941437.351392:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941437.351394:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941437.351394:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000001:2.0:1773941437.351397:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.351400:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941437.351400:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c91 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941437.351407:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 10000000:00000001:1.0:1773941437.351410:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000010:3.0:1773941437.351412:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0800. 10000000:00010000:1.0:1773941437.351417:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c91 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941437.351419:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941437.351440:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 10000000:01000000:1.0:1773941437.351441:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:3.0:1773941437.351448:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34198 10000000:00000040:1.0:1773941437.351448:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941437.351452:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941437.351454:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34198. 00010000:00010000:1.0:1773941437.351460:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941437.351465:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941437.351468:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941437.351469:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941437.351472:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941437.351474:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941437.351477:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f000 with handle 0xa33a5be14eae1c91 from hash 00010000:00000001:1.0:1773941437.351480:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941437.351482:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941437.351483:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941437.351490:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941437.351492:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941437.351494:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941437.351497:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941437.351498:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941437.351499:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941437.351501:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941437.351505:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c91 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941437.351512:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941437.351513:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941437.351515:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941437.351518:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:3.0:1773941437.351519:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941437.351521:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941437.351525:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000200:3.0:1773941437.351527:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62680 x1860112241147904/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941447 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941437.351529:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941437.351534:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1c91 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941437.351538:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941437.351544:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62680 x1860112241147904/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941447 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941437.351571:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941437.351573:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000040:1.0:1773941437.351574:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ecd00 count: 0 00010000:00000001:1.0:1773941437.351590:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941437.351594:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941437.351598:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941437.351606:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62a00 x1860112241147776/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941453 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941437.351613:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941437.351616:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941437.351619:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0737400. 02000000:00000001:1.0:1773941437.351622:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941437.351625:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941437.351626:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941437.351629:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941437.351631:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7400. 02000000:00000001:1.0:1773941437.351634:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941437.351636:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941437.351644:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62a00. 00000100:00000001:1.0:1773941437.351647:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941437.351649:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941437.351651:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941437.351654:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941437.351656:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941437.351660:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0b00. 00000020:00000010:1.0:1773941437.351664:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7400. 00000020:00000010:1.0:1773941437.351669:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b7800. 00000040:00000010:1.0:1773941437.351673:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7a00. 00000020:00000010:1.0:1773941437.351677:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941437.351681:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941437.351685:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941437.351690:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6e00. 20000000:00000010:1.0:1773941437.351694:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b7000. 10000000:00000040:1.0:1773941437.351699:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941437.351701:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941437.351735:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b7800. 00000040:00000010:1.0:1773941437.351740:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7a00. 00000020:00000010:1.0:1773941437.351744:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941437.351749:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941437.351752:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941437.351757:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6e00. 20000000:00000010:1.0:1773941437.351759:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941437.351762:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7400. 10000000:00000010:1.0:1773941437.351765:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0b00. 10000000:01000000:1.0:1773941437.351767:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941437.351770:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941437.351772:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941437.351774:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941437.351776:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941437.351777:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941437.351780:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941437.352789:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.352795:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941437.352797:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.352805:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941437.352810:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941437.352814:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941437.352818:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941437.352821:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941437.352823:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941437.352826:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604000 00000400:00000200:3.0:1773941437.352829:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941437.352833:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xbe5 [1] + 0 00000800:00000001:3.0:1773941437.352837:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.352841:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941437.352843:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941437.352846:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941437.352849:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941437.352853:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62680 x1860112241147904/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941447 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941437.352860:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d62680 x1860112241147904/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941447 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941437.352875:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941437.352879:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941437.352881:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941437.352898:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941437.352901:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941437.352905:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941437.352907:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941437.352909:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941437.352914:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941437.352915:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941437.352918:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941437.352923:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62680 x1860112241147904/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941447 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941437.352931:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62680 x1860112241147904/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941447 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941437.352937:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941437.352938:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941437.352939:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941437.352941:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941437.352942:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941437.352943:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941437.352945:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941437.352948:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=16 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941437.352971:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941437.352973:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941437.352974:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941437.352975:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941437.352977:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.352979:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941437.352982:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62680 x1860112241147904/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941447 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941437.352987:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941437.352989:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941437.352990:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941437.352992:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941437.352996:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941437.352998:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941437.352999:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941437.353000:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941437.353001:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941437.353003:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941437.353005:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941437.353008:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941437 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941437.353009:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941437.353015:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941437.353017:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941437.353018:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941437.353020:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.353021:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941437.353023:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941437.353025:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941437.353026:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941437.353029:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941437.353031:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941437.353033:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941437.353265:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.353274:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941437.353279:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62680 x1860112241147904/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941447 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941437.353280:0:12163:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941437.353286:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241147904:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941437.353293:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941437.353296:0:12163:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:2.0:1773941437.353297:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1773941437.353301:0:12163:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:2.0:1773941437.353303:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62680 x1860112241147904/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941447 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941437.353304:0:12163:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941437.353307:0:12163:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941437.353311:0:12163:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941437.353313:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:1.0:1773941437.353314:0:12163:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941437.353316:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941437.353318:0:12163:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941437.353320:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce4000. 00000001:00000001:1.0:1773941437.353321:0:12163:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941437.353323:0:12163:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941437.353325:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941437.353326:0:12163:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941437.353328:0:2351:0:(genops.c:1236:class_import_put()) Process entered 10000000:00000001:1.0:1773941437.353329:0:12163:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941437.353331:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941437.353332:0:12163:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:2.0:1773941437.353336:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:1.0:1773941437.353337:0:12163:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:2.0:1773941437.353338:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0736800. 00000100:00000001:1.0:1773941437.353342:0:12163:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:2.0:1773941437.353343:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941437.353345:0:12163:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:2.0:1773941437.353346:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941437.353348:0:12163:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941437.353349:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62680. 00000100:00080000:1.0:1773941437.353352:0:12163:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941437.353354:0:12163:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941437.353355:0:12163:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941437.353356:0:12163:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941437.353372:0:12163:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941437.353375:0:12163:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.353377:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941437.353377:0:12163:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941437.353381:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:02000000:1.0:1773941437.353383:0:12163:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941437.353384:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941437.353386:0:12163:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941437.353389:0:12163:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941437.353391:0:12163:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941437.353395:0:12163:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941437.353398:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941437.353398:0:12163:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.353401:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941437.353404:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941437.353407:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941437.358738:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ecd00. 00010000:00000010:1.1:1773941437.358747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f000. 00000020:00000001:0.0:1773941438.303909:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941438.303920:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941438.303927:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941438.687787:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941438.687797:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941438.687801:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941438.687804:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941438.687805:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941438.687807:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941438.687809:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941438.687812:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941438.687814:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941438.687817:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941438.687818:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941438.687832:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941438.687838:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941438.687861:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941438.687864:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941438.687870:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941438.687877:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2a00. 00000100:00000001:3.0:1773941438.687885:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941438.687890:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941438.687892:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941438.687901:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962000. 00010000:00000001:3.0:1773941438.687907:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941438.687910:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81484600. 00000020:00000010:3.0:1773941438.687914:0:12144:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff8b5b9020a580. 00000020:00000001:3.0:1773941438.687918:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941438.687920:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81484600 with handle 0xa33a5be14eae1c98 to hash 00000020:00000001:3.0:1773941438.687922:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941438.687923:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:3.0:1773941438.687926:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:3.0:1773941438.687944:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941438.687948:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941438.687951:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c98 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941438.687962:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c98 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941438.687973:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c98 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941438.687978:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941438.687980:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941438.687987:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e9c0. 00000100:00000001:3.0:1773941438.687988:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387402176 : -128249322149440 : ffff8b5ba0b5e9c0) 00000100:00000001:3.0:1773941438.687991:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941438.687992:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941438.687998:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941438.688007:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941438.688009:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941438.688011:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941438.688022:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241148032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941438.688030:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941438.688032:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941438.688042:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941438.688045:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941438.688049:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941438.688052:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241148032:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941438.688056:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941438.688062:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941438.688064:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941438.688067:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941438.688071:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941438.688073:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941438.688082:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941438.688085:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab4d0. 00000400:00000010:3.0:1773941438.688093:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a7f8. 00000100:00000200:3.0:1773941438.688100:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241148032, portal 25 00000100:00000001:3.0:1773941438.688101:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941438.688102:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941438.688107:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241148032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941454 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941438.688113:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941438.688115:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941438.688117:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a550. 00000100:00000200:3.0:1773941438.688120:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241148032, offset 0 00000400:00000200:3.0:1773941438.688125:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941438.688134:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941438.688142:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941438.688147:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941438.688154:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941438.688158:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941438.688160:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941438.688163:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[30, 29] 00000400:00000200:3.0:1773941438.688166:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941438.688170:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [136:136:256:4294967295] 192.168.204.153@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:3.0:1773941438.688173:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941438.688183:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941438.688188:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941438.688193:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69700. 00000800:00000200:3.0:1773941438.688199:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941438.688207:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941438.688211:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69700 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941438.688233:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941438.688236:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941438.688239:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294879051 00000100:00000001:3.0:1773941438.688245:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294877714 00000100:00000001:3.0:1773941438.688250:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294879051 00000100:00000001:3.0:1773941438.688253:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294877714 00000100:00000001:3.0:1773941438.688255:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941438.688259:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941438.688261:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941438.688264:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941438.688267:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941438.688271:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941438.688275:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941438.688407:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941438.688414:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69700. 00000400:00000200:2.0:1773941438.688421:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941438.688428:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941438.688435:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a550 00000400:00000010:2.0:1773941438.688437:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a550. 00000100:00000001:2.0:1773941438.688441:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941438.688447:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241148032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941454 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941438.688469:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941438.688472:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241148032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941454 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941438.688477:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941438.688479:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941438.689471:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941438.689482:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941438.689485:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941438.689486:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941438.689492:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941438.689536:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941438.689542:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941438.689545:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941438.689550:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941438.689553:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604080 00000400:00000200:3.0:1773941438.689556:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941438.689561:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xbf5 [1] + 224 00000800:00000001:3.0:1773941438.689567:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941438.689574:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941438.689577:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941438.689581:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941438.689586:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941438.689592:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241148032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941454 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941438.689603:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241148032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941454 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941438.689629:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941438.689636:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941438.689640:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941438.689684:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941438.689692:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941438.689693:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941438.689696:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab4d0. 00000400:00000200:1.0:1773941438.689722:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a7f8 00000400:00000010:1.0:1773941438.689724:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a7f8. 00000100:00000001:1.0:1773941438.689728:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941438.689740:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241148032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941454 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941438.689748:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241148032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941454 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941438.689754:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941438.689755:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941438.689758:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941438.689760:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941438.689762:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941438.689764:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941438.689766:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941438.689772:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=17 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941438.689774:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941438.689777:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941438.689781:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941438.689783:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941438.689786:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941438.689788:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941438.689791:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941438.689795:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941438.689798:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941438.689800:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941438.689801:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941438.689803:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941438.689806:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241148032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941454 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941438.689810:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941438.689811:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941438.689813:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941438.689815:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941438.689818:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941438.689819:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941438.689823:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941438.689824:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941438.689825:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941438.689826:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941438.689827:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941438.689830:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941438.689831:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941438.689834:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941438.689836:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 207 00000400:00000200:1.0:1773941438.689840:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941438.689843:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941438.689846:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941438.689849:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941438.689851:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941438.689852:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941438.689855:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941438.689857:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941438.689858:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941438.689861:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941438.689862:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941438.689864:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941438.689866:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941438.689867:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941438.689868:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941438.689869:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941438.689871:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941438.689874:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941438.689876:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941438.689877:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941438.689880:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941438.689882:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941438.689884:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941438.689888:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941438.689892:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62a00. 00000020:00000040:1.0:1773941438.689894:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941438.689897:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941438.689899:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941438.689901:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941438.689902:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941438.689904:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941438.689907:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737400. 00000100:00000001:1.0:1773941438.689910:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941438.689915:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d62a00 x1860112241148160/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941438.689921:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941438.689924:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941438.689941:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000acb4b66b] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d62a00 x1860112241148160/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941438.689965:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941438.689967:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941438.689969:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941438.689972:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941438.689975:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941438.689977:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941438.689979:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941438.689982:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941438.689984:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941438.689990:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241148032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941454 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941438.690000:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941438.690004:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941438.690006:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941438.690009:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941438.690010:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1773941438.690016:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241148032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941454 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941438.690019:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941438.690026:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941438.690037:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62a00 x1860112241148160/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941438.690042:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241148032:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941438.690060:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941438.690063:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941438.690065:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941438.690067:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941438.690072:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241148032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941454 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941438.690075:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941438.690081:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941438.690091:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941438.690093:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941438.690096:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1773941438.690097:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e9c0. 02000000:00000001:2.0:1773941438.690104:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941438.690105:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941438.690111:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:2.0:1773941438.690112:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241148160:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941438.690122:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:2.0:1773941438.690123:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941438.690131:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:2.0:1773941438.690132:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941438.690136:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941438.690136:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941438.690140:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484600 refcount=4 02000000:00000001:2.0:1773941438.690142:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:1.0:1773941438.690149:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:1.0:1773941438.690153:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 02000000:00000010:2.0:1773941438.690154:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce4000. 02000000:00000001:2.0:1773941438.690167:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941438.690168:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c98 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941438.690182:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:2.0:1773941438.690185:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941438.690192:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c98 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941438.690193:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941438.690204:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00000001:1.0:1773941438.690205:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941438.690209:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c98 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941438.690213:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241148160, portal 25 00000100:00000001:2.0:1773941438.690224:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941438.690224:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941438.690227:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121080320 : -128249588471296 : ffff8b5b90d62a00) 00010000:00000001:1.0:1773941438.690233:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000040:2.0:1773941438.690234:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62a00 x1860112241148160/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941448 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941438.690236:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c98 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941438.690280:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941438.690280:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941438.690283:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941438.690287:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941438.690289:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c98 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941438.690301:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941438.690305:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81484600 already unused, calling callback (ffffffffc1482c20) 00000400:00000010:2.0:1773941438.690306:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f346e8. 10000000:00000001:1.0:1773941438.690314:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000200:2.0:1773941438.690315:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241148160, offset 0 10000000:00010000:1.0:1773941438.690318:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c98 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941438.690329:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941438.690336:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941438.690338:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941438.690339:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941438.690340:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484600 refcount=5 00000400:00000200:2.0:1773941438.690343:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:1.0:1773941438.690347:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:1.0:1773941438.690350:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00000400:00000200:2.0:1773941438.690353:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:1.0:1773941438.690358:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:2.0:1773941438.690360:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:1.0:1773941438.690362:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c98 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941438.690369:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941438.690378:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000001:1.0:1773941438.690378:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941438.690384:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c98 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941438.690387:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941438.690400:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[30, 31] 10000000:01000000:1.0:1773941438.690411:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941438.690416:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 10000000:00000040:1.0:1773941438.690422:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941438.690425:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941438.690430:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [137:137:256:4294967295] 192.168.204.143@tcp LPNI seq info [32:32:8:4294967295] 00010000:00010000:1.0:1773941438.690433:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:2.0:1773941438.690438:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941438.690439:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941438.690452:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941438.690452:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941438.690463:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941438.690466:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941438.690469:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941438.690474:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000020:00000040:1.0:1773941438.690476:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81484600 with handle 0xa33a5be14eae1c98 from hash 00000800:00000010:2.0:1773941438.690480:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0200. 00010000:00000001:1.0:1773941438.690481:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941438.690486:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941438.690489:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941438.690491:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941438.690497:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000800:00000200:2.0:1773941438.690499:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941438.690500:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941438.690508:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941438.690511:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941438.690517:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941438.690518:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 10000000:00000001:1.0:1773941438.690528:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941438.690529:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0200 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:1.0:1773941438.690538:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c98 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941438.690543:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941438.690544:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941438.690545:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941438.690546:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941438.690547:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941438.690548:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941438.690549:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941438.690551:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941438.690551:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1c98 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941438.690568:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000040:1.0:1773941438.690574:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962000 count: 0 00000100:00000001:2.0:1773941438.690575:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294879053 00000100:00000001:2.0:1773941438.690593:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294879051 00000100:00000001:2.0:1773941438.690602:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294879053 00010000:00000001:1.0:1773941438.690604:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941438.690608:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294879051 00010000:00000001:1.0:1773941438.690608:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941438.690619:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941438.690620:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941438.690625:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941438.690629:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000040:1.0:1773941438.690630:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241148032/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941454 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941438.690632:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1773941438.690640:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0200. 00000100:00000001:2.0:1773941438.690644:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941438.690654:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941438.690665:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941438.690666:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000400:00000200:3.0:1773941438.690671:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941438.690672:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:1.0:1773941438.690674:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:3.0:1773941438.690677:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f346e8 00000100:00000001:2.0:1773941438.690678:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1773941438.690684:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f346e8. 00000100:00000001:2.0:1773941438.690685:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 02000000:00000010:1.0:1773941438.690686:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 00000100:00000001:2.0:1773941438.690690:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941438.690694:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941438.690695:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1773941438.690695:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941438.690750:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941438.690754:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941438.690757:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941438.690760:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2a00. 02000000:00000001:1.0:1773941438.690764:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941438.690766:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941438.690768:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:1.0:1773941438.690772:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941438.690774:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1773941438.690776:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62a00 x1860112241148160/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941448 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941438.690778:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941438.690783:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941438.690791:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941438.690795:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0b00. 00000020:00000010:1.0:1773941438.690803:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7400. 00000100:00000001:3.0:1773941438.690806:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000010:1.0:1773941438.690811:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b7000. 00000100:00000040:3.0:1773941438.690819:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62a00 x1860112241148160/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941448 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000040:00000010:1.0:1773941438.690819:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941438.690828:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941438.690835:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941438.690841:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941438.690846:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7a00. 20000000:00000010:1.0:1773941438.690853:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b7800. 00000100:00000001:3.0:1773941438.690866:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:1.0:1773941438.690866:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000100:00000001:3.0:1773941438.690879:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000001:1.0:1773941438.690879:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941438.690890:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b7000. 00000040:00000010:1.0:1773941438.690896:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941438.690899:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941438.690903:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941438.690905:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941438.690910:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7a00. 20000000:00000010:1.0:1773941438.690915:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941438.690919:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7400. 10000000:00000010:1.0:1773941438.690921:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0b00. 10000000:01000000:1.0:1773941438.690923:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941438.690946:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941438.690949:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941438.690951:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941438.690953:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941438.690955:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941438.690958:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941438.691899:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941438.691904:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941438.691906:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941438.691909:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941438.691914:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941438.691920:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941438.691925:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941438.691940:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941438.692005:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941438.692008:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604100 00000400:00000200:3.0:1773941438.692011:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941438.692013:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xc05 [1] + 0 00000800:00000001:3.0:1773941438.692017:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941438.692022:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941438.692024:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941438.692026:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941438.692029:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941438.692032:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62a00 x1860112241148160/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941448 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941438.692039:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d62a00 x1860112241148160/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941448 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941438.692059:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941438.692065:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941438.692068:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941438.692100:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941438.692103:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941438.692107:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941438.692108:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941438.692110:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941438.692114:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941438.692115:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941438.692117:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941438.692122:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62a00 x1860112241148160/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941448 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941438.692130:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62a00 x1860112241148160/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941448 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941438.692135:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941438.692136:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941438.692137:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941438.692139:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941438.692141:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941438.692143:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941438.692144:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941438.692149:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=17 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941438.692152:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941438.692154:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941438.692155:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941438.692157:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941438.692159:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941438.692161:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941438.692164:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62a00 x1860112241148160/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941448 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941438.692170:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941438.692172:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941438.692180:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941438.692182:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941438.692187:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941438.692190:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941438.692191:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941438.692193:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941438.692195:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941438.692197:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941438.692199:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941438.692202:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941438 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941438.692204:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941438.692209:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941438.692210:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941438.692213:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941438.692214:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941438.692216:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941438.692218:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941438.692220:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941438.692222:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941438.692224:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941438.692227:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941438.692229:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941438.692457:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941438.692460:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941438.692464:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62a00 x1860112241148160/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941448 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941438.692472:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241148160:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941438.692477:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941438.692477:0:12164:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941438.692482:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941438.692485:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62a00 x1860112241148160/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941448 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941438.692491:0:12164:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:1.0:1773941438.692502:0:12164:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941438.692506:0:12164:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941438.692511:0:12164:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941438.692519:0:12164:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941438.692524:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:1.0:1773941438.692525:0:12164:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941438.692529:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941438.692529:0:12164:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941438.692535:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce4000. 00000001:00000001:1.0:1773941438.692535:0:12164:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941438.692542:0:12164:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941438.692544:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941438.692546:0:12164:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941438.692547:0:2351:0:(genops.c:1236:class_import_put()) Process entered 10000000:00000001:1.0:1773941438.692553:0:12164:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941438.692554:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941438.692563:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941438.692563:0:12164:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:2.0:1773941438.692569:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737400. 00000100:00080000:1.0:1773941438.692571:0:12164:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:2.0:1773941438.692577:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941438.692577:0:12164:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:2.0:1773941438.692581:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941438.692581:0:12164:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941438.692592:0:12164:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941438.692593:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62a00. 00000100:00080000:1.0:1773941438.692603:0:12164:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941438.692604:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941438.692608:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941438.692609:0:12164:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941438.692611:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:1.0:1773941438.692612:0:12164:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941438.692616:0:12164:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:2.0:1773941438.692617:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941438.692619:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941438.692622:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941438.692623:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941438.692632:0:12164:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941438.692634:0:12164:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941438.692636:0:12164:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941438.692639:0:12164:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941438.692647:0:12164:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941438.692650:0:12164:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941438.692652:0:12164:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941438.692654:0:12164:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941438.692656:0:12164:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941438.696743:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962000. 00010000:00000010:1.1:1773941438.696750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81484600. 00000020:00000001:3.0:1773941439.328037:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941439.328043:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941439.328047:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941440.287850:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941440.287862:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941440.287895:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941440.287911:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941440.287913:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941440.287915:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941440.287919:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941440.287922:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941440.287925:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941440.287928:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941440.287930:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941440.287940:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275f80. 00000020:00000040:3.0:1773941440.287947:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941440.287952:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941440.287957:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941440.287963:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941440.287972:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2600. 00000100:00000001:3.0:1773941440.287980:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941440.287985:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941440.287989:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941440.287998:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962100. 00010000:00000001:3.0:1773941440.288005:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941440.288009:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485200. 00000020:00000001:3.0:1773941440.288023:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941440.288027:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485200 with handle 0xa33a5be14eae1c9f to hash 00000020:00000001:3.0:1773941440.288030:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941440.288032:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00010000:00000001:3.0:1773941440.288036:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00010000:00000001:3.0:1773941440.288041:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941440.288045:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941440.288051:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1c9f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941440.288064:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1c9f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941440.288078:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1c9f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941440.288091:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941440.288094:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941440.288102:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941440.288104:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941440.288107:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941440.288109:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109625216 : -128249599926400 : ffff8b5b90275f80) 00000100:00000001:3.0:1773941440.288115:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941440.288121:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941440.288124:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941440.288126:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941440.288138:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275f80 x1860112241148288/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941440.288148:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941440.288151:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941440.288156:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941440.288159:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941440.288162:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941440.288165:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275f80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241148288:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941440.288170:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941440.288176:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941440.288179:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941440.288182:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941440.288187:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90263000. 02000000:00000001:3.0:1773941440.288189:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941440.288198:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941440.288203:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941440.288211:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a220. 00000100:00000200:3.0:1773941440.288219:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241148288, portal 25 00000100:00000001:3.0:1773941440.288221:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941440.288223:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109625216 : -128249599926400 : ffff8b5b90275f80) 00000100:00000040:3.0:1773941440.288228:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275f80 x1860112241148288/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941456 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941440.288246:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941440.288248:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941440.288251:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6af68. 00000100:00000200:3.0:1773941440.288254:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241148288, offset 0 00000400:00000200:3.0:1773941440.288259:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941440.288268:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941440.288276:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941440.288281:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941440.288285:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941440.288289:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941440.288291:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941440.288294:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[32, 31] 00000400:00000200:3.0:1773941440.288297:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941440.288300:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [138:138:256:4294967295] 192.168.204.153@tcp LPNI seq info [33:33:8:4294967295] 00000400:00000200:3.0:1773941440.288304:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941440.288312:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941440.288318:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941440.288323:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69c00. 00000800:00000200:3.0:1773941440.288329:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941440.288337:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941440.288341:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69c00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941440.288364:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941440.288367:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941440.288370:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294880651 00000100:00000001:3.0:1773941440.288377:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294879053 00000100:00000001:3.0:1773941440.288382:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294880651 00000100:00000001:3.0:1773941440.288385:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294879053 00000100:00000001:3.0:1773941440.288388:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941440.288391:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941440.288399:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941440.288402:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941440.288406:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941440.288410:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941440.288415:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941440.288516:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941440.288521:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69c00. 00000400:00000200:2.0:1773941440.288527:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941440.288533:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941440.288537:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6af68 00000400:00000010:2.0:1773941440.288539:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6af68. 00000100:00000001:2.0:1773941440.288545:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941440.288549:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275f80 x1860112241148288/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941456 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941440.288558:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941440.288561:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275f80 x1860112241148288/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941456 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941440.288567:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941440.288568:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941440.289627:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941440.289640:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941440.289644:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941440.289647:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941440.289653:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941440.289659:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941440.289727:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941440.289733:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941440.289738:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941440.289743:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604180 00000400:00000200:3.0:1773941440.289747:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941440.289753:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xc15 [1] + 224 00000800:00000001:3.0:1773941440.289761:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941440.289770:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941440.289773:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941440.289776:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941440.289782:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941440.289787:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275f80 x1860112241148288/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941456 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941440.289799:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275f80 x1860112241148288/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941456 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941440.289820:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941440.289829:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941440.289833:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941440.289872:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941440.289879:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941440.289880:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941440.289883:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:1.0:1773941440.289887:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a220 00000400:00000010:1.0:1773941440.289888:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a220. 00000100:00000001:1.0:1773941440.289891:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941440.289896:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275f80 x1860112241148288/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941456 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941440.289918:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275f80 x1860112241148288/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941456 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941440.289924:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941440.289926:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941440.289929:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941440.289931:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941440.289934:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941440.289936:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941440.289938:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941440.289944:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=19 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941440.289947:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941440.289950:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941440.289954:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941440.293644:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941440.293649:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941440.293652:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941440.293657:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941440.293661:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941440.297735:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941440.297738:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941440.297740:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941440.297743:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941440.297751:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275f80 x1860112241148288/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941456 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941440.297770:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941440.297771:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941440.297773:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941440.297776:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941440.297778:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941440.297780:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941440.297781:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941440.297782:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941440.297783:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941440.297784:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941440.297785:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941440.297787:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941440.297789:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941440.297792:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941440.297795:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 209 00000400:00000200:1.0:1773941440.297800:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941440.297806:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941440.297810:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941440.297814:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941440.297817:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941440.297818:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941440.297822:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941440.297824:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941440.297825:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941440.297828:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941440.297830:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941440.297831:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941440.297833:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941440.297834:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941440.297836:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941440.297837:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941440.297839:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941440.297845:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941440.297848:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941440.297850:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941440.297853:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941440.297855:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941440.297857:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941440.297859:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941440.297864:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941440.297867:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941440.297870:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941440.297871:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941440.297874:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941440.297876:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941440.297879:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941440.297883:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734000. 00000100:00000001:1.0:1773941440.297887:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941440.297892:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63800 x1860112241148416/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941440.297920:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941440.297923:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941440.297928:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc928f9c] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63800 x1860112241148416/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941440.297944:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941440.297946:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941440.297948:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941440.297950:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941440.297952:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941440.297954:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941440.297956:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941440.297958:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941440.297961:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941440.297965:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275f80 x1860112241148288/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941456 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941440.297967:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941440.297972:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941440.297983:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941440.297984:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941440.297987:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941440.297990:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:2.0:1773941440.297991:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241148416/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941440.297993:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941440.298003:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275f80 x1860112241148288/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941456 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941440.298005:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941440.298007:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941440.298013:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941440.298016:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1773941440.298016:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275f80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241148288:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941440.298023:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941440.298025:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941440.298029:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:2.0:1773941440.298030:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241148416:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941440.298031:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941440.298035:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941440.298041:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1773941440.298041:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275f80 x1860112241148288/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941456 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941440.298048:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941440.298049:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941440.298053:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941440.298053:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:1.0:1773941440.298058:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:2.0:1773941440.298059:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941440.298061:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941440.298065:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000010:2.0:1773941440.298066:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce7400. 00010000:00000001:1.0:1773941440.298068:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941440.298072:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:2.0:1773941440.298073:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941440.298074:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485200 refcount=4 00000020:00000001:1.0:1773941440.298078:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00010000:00000001:1.0:1773941440.298081:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00000400:00000200:2.0:1773941440.298084:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941440.298087:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1c9f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941440.298090:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00010000:00000001:1.0:1773941440.298098:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000010:2.0:1773941440.298099:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00010000:1.0:1773941440.298103:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1c9f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941440.298107:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241148416, portal 25 00000100:00000001:2.0:1773941440.298112:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941440.298115:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00010000:00000001:1.0:1773941440.298119:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941440.298123:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1c9f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941440.298126:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241148416/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941450 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941440.298130:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941440.298134:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941440.298137:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1c9f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941440.298145:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941440.298148:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000040:2.0:1773941440.298152:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941440.298153:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1773941440.298158:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34e58. 00010000:00000001:1.0:1773941440.298158:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941440.298160:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1c9f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941440.298163:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241148416, offset 0 00010000:00010000:1.0:1773941440.298170:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485200 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941440.298172:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941440.298175:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 10000000:00010000:1.0:1773941440.298177:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1c9f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941440.298183:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:2.0:1773941440.298184:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941440.298185:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941440.298188:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941440.298191:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485200 refcount=5 00000020:00000001:1.0:1773941440.298192:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00000400:00000200:2.0:1773941440.298195:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:1.0:1773941440.298195:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00010000:00000001:1.0:1773941440.298202:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:2.0:1773941440.298204:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:1.0:1773941440.298205:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1c9f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941440.298210:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 10000000:00000001:1.0:1773941440.298213:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941440.298216:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00010000:1.0:1773941440.298220:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1c9f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941440.298231:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:01000000:1.0:1773941440.298232:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941440.298239:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[32, 33] 10000000:00000040:1.0:1773941440.298240:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941440.298243:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941440.298248:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00010000:1.0:1773941440.298250:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941440.298255:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941440.298256:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [139:139:256:4294967295] 192.168.204.143@tcp LPNI seq info [34:34:8:4294967295] 00010000:00000001:1.0:1773941440.298261:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:2.0:1773941440.298266:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941440.298266:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941440.298271:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941440.298273:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941440.298280:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000020:00000040:1.0:1773941440.298280:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485200 with handle 0xa33a5be14eae1c9f from hash 00010000:00000001:1.0:1773941440.298286:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941440.298290:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941440.298291:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941440.298293:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941440.298296:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000800:00000010:2.0:1773941440.298298:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0c00. 00010000:00000001:1.0:1773941440.298298:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941440.298302:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941440.298305:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941440.298306:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941440.298308:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941440.298310:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00000001:1.0:1773941440.298311:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941440.298317:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1c9f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941440.298324:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941440.298325:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941440.298326:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941440.298329:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000800:00000200:2.0:1773941440.298332:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0c00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941440.298332:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941440.298334:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941440.298338:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941440.298340:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941440.298342:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1c9f lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941440.298352:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962100 count: 0 00000100:00000001:2.0:1773941440.298353:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941440.298359:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941440.298364:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294880661 00000100:00000001:2.0:1773941440.298374:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294880651 00000100:00000001:2.0:1773941440.298382:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294880661 00010000:00000001:1.0:1773941440.298382:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941440.298387:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941440.298389:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294880651 00000100:00000001:1.0:1773941440.298393:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941440.298396:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1773941440.298400:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275f80 x1860112241148288/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941456 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941440.298402:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941440.298408:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941440.298415:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941440.298417:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941440.298422:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941440.298425:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000010:1.0:1773941440.298425:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90263000. 00000100:00000001:2.0:1773941440.298429:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:1.0:1773941440.298433:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1773941440.298434:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941440.298438:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1773941440.298439:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941440.298441:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941440.298443:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000020:00000040:1.0:1773941440.298443:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941440.298446:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1773941440.298451:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0c00. 00000020:00000001:1.0:1773941440.298451:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941440.298457:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2600. 00000400:00000200:3.0:1773941440.298459:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:1.0:1773941440.298463:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941440.298466:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000400:00000200:3.0:1773941440.298469:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000010:1.0:1773941440.298470:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275f80. 00000100:00000001:1.0:1773941440.298474:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941440.298477:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941440.298481:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:3.0:1773941440.298483:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34e58 10000000:01000000:1.0:1773941440.298484:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000400:00000010:3.0:1773941440.298489:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34e58. 10000000:00000001:1.0:1773941440.298489:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941440.298493:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0b00. 00000020:00000010:1.0:1773941440.298497:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7400. 00000020:00000010:1.0:1773941440.298501:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b7800. 00000040:00000010:1.0:1773941440.298505:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7a00. 00000020:00000010:1.0:1773941440.298509:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941440.298513:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b6800. 80000000:00000010:1.0:1773941440.298518:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941440.298522:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6e00. 20000000:00000010:1.0:1773941440.298526:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b7000. 10000000:00000040:1.0:1773941440.298529:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941440.298532:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941440.298535:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:1.0:1773941440.298537:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b7800. 00000100:00000200:3.0:1773941440.298542:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241148416/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941450 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000040:00000010:1.0:1773941440.298544:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7a00. 00000100:00000001:3.0:1773941440.298551:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000010:1.0:1773941440.298554:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000100:00000040:3.0:1773941440.298555:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241148416/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941450 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941440.298561:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b6800. 00000100:00000001:3.0:1773941440.298564:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 80000000:00000010:1.0:1773941440.298566:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00000100:00000001:3.0:1773941440.298567:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00040000:00000010:1.0:1773941440.298574:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6e00. 20000000:00000010:1.0:1773941440.298578:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941440.298580:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7400. 10000000:00000010:1.0:1773941440.298583:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0b00. 10000000:01000000:1.0:1773941440.298585:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941440.298588:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941440.298590:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941440.298592:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941440.298594:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941440.298595:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941440.298597:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941440.303112:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941440.303120:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941440.303122:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941440.303124:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941440.303128:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941440.303132:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941440.303136:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941440.303139:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941440.303142:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941440.303145:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604200 00000400:00000200:3.0:1773941440.303148:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941440.303197:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xc25 [1] + 0 00000800:00000001:3.0:1773941440.303203:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941440.303209:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941440.303211:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941440.303213:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941440.303216:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941440.303220:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241148416/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941450 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941440.303236:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63800 x1860112241148416/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941450 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941440.303248:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941440.303259:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941440.303262:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941440.303293:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941440.303297:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941440.303302:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941440.303304:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941440.303306:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941440.303318:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941440.303320:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941440.303324:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941440.303330:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241148416/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941450 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941440.303342:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241148416/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941450 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941440.303351:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941440.303353:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941440.303356:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941440.303359:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941440.303362:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941440.303364:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941440.303367:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941440.303373:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=19 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941440.303377:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941440.303380:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941440.303382:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941440.303385:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941440.303387:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941440.303391:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941440.303396:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241148416/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941450 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941440.303404:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941440.303406:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941440.303408:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941440.303411:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941440.303416:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941440.303419:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941440.303421:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941440.303424:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941440.303426:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941440.303428:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941440.303431:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941440.303445:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941440 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941440.303449:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941440.303456:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941440.303459:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941440.303461:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941440.303463:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941440.303465:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941440.303469:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941440.303472:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941440.303475:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941440.303479:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941440.303483:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941440.303486:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00010000:00000010:1.1:1773941440.303729:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962100. 00010000:00000010:1.1:1773941440.303736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485200. 00000100:00000001:2.0:1773941440.303743:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941440.303750:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941440.303750:0:12165:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941440.303758:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241148416/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941450 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941440.303761:0:12165:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:1.0:1773941440.303768:0:12165:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:2.0:1773941440.303778:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241148416:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941440.303789:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941440.303792:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941440.303797:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241148416/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941450 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941440.303802:0:12165:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941440.303806:0:12165:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941440.303810:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:1.0:1773941440.303811:0:12165:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941440.303816:0:12165:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941440.303817:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941440.303819:0:12165:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941440.303821:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce7400. 00000001:00000001:1.0:1773941440.303822:0:12165:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941440.303825:0:12165:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941440.303828:0:12165:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941440.303829:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:00000001:1.0:1773941440.303830:0:12165:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941440.303834:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941440.303834:0:12165:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:2.0:1773941440.303839:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00080000:1.0:1773941440.303840:0:12165:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000020:00000001:2.0:1773941440.303846:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941440.303846:0:12165:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000010:2.0:1773941440.303851:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734000. 00000100:00000001:1.0:1773941440.303851:0:12165:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941440.303854:0:12165:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941440.303857:0:12165:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 02000000:00000001:2.0:1773941440.303858:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941440.303859:0:12165:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941440.303861:0:12165:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941440.303862:0:12165:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 02000000:00000001:2.0:1773941440.303868:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941440.303871:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:2.0:1773941440.303876:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941440.303878:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941440.303881:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941440.303885:0:12165:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:2.0:1773941440.303888:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941440.303891:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941440.303896:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000001:1.0:1773941440.303906:0:12165:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941440.303910:0:12165:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941440.303913:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:02000000:1.0:1773941440.303918:0:12165:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941440.306948:0:12165:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941440.306951:0:12165:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941440.306953:0:12165:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941440.306955:0:12165:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941440.306957:0:12165:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941440.351939:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941440.351944:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941440.351948:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941441.375936:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941441.375942:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941441.375946:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941441.887828:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941441.887835:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941441.887838:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941441.887840:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941441.887841:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941441.887842:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941441.887845:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941441.887846:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941441.887848:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941441.887850:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941441.887852:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941441.887861:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90276300. 00000020:00000040:3.0:1773941441.887865:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941441.887891:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941441.887895:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941441.887900:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941441.887910:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e3000. 00000100:00000001:3.0:1773941441.887917:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941441.887921:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941441.887924:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941441.887932:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962600. 00010000:00000001:3.0:1773941441.887938:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941441.887942:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81484200. 00000020:00000001:3.0:1773941441.887947:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941441.887950:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81484200 with handle 0xa33a5be14eae1ca6 to hash 00000020:00000001:3.0:1773941441.887952:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941441.887954:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860122112 : -128249849429504 : ffff8b5b81484200) 00010000:00000001:3.0:1773941441.887957:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860122112 : -128249849429504 : ffff8b5b81484200) 00010000:00000001:3.0:1773941441.887962:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941441.887966:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941441.887970:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1ca6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941441.887994:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1ca6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941441.888003:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1ca6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941441.888008:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941441.888010:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941441.888015:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e300. 00000100:00000001:3.0:1773941441.888016:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387400448 : -128249322151168 : ffff8b5ba0b5e300) 00000100:00000001:3.0:1773941441.888018:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941441.888019:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109626112 : -128249599925504 : ffff8b5b90276300) 00000100:00000001:3.0:1773941441.888023:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941441.888027:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941441.888029:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941441.888031:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941441.888039:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90276300 x1860112241148544/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941441.888045:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941441.888047:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941441.888050:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941441.888052:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941441.888054:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941441.888056:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241148544:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941441.888059:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941441.888064:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941441.888066:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941441.888068:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941441.888071:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90263000. 02000000:00000001:3.0:1773941441.888073:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941441.888079:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941441.888085:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab058. 00000400:00000010:3.0:1773941441.888091:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6ad48. 00000100:00000200:3.0:1773941441.888096:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241148544, portal 25 00000100:00000001:3.0:1773941441.888097:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941441.888098:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109626112 : -128249599925504 : ffff8b5b90276300) 00000100:00000040:3.0:1773941441.888102:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90276300 x1860112241148544/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941457 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941441.888108:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941441.888110:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941441.888112:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6abb0. 00000100:00000200:3.0:1773941441.888114:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241148544, offset 0 00000400:00000200:3.0:1773941441.888119:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941441.888125:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941441.888132:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941441.888136:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941441.888140:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941441.888143:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941441.888145:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941441.888148:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[34, 33] 00000400:00000200:3.0:1773941441.888150:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941441.888153:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [140:140:256:4294967295] 192.168.204.153@tcp LPNI seq info [35:35:8:4294967295] 00000400:00000200:3.0:1773941441.888156:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941441.888166:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941441.888172:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941441.888177:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69600. 00000800:00000200:3.0:1773941441.888184:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941441.888192:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941441.888197:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941441.888220:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941441.888225:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941441.888228:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294882251 00000100:00000001:3.0:1773941441.888240:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294880661 00000100:00000001:3.0:1773941441.888246:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294882251 00000100:00000001:3.0:1773941441.888249:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294880661 00000100:00000001:3.0:1773941441.888251:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941441.888254:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941441.888256:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941441.888257:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941441.888259:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 11 seconds 00000100:00000001:3.0:1773941441.888261:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941441.888264:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941441.888339:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941441.888344:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69600. 00000400:00000200:2.0:1773941441.888348:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941441.888353:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941441.888357:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6abb0 00000400:00000010:2.0:1773941441.888359:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6abb0. 00000100:00000001:2.0:1773941441.888363:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941441.888367:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90276300 x1860112241148544/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941457 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941441.888376:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941441.888378:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90276300 x1860112241148544/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941457 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941441.888384:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941441.888385:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941441.889528:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941441.889537:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941441.889539:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941441.889541:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941441.889546:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941441.889550:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941441.889556:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941441.889620:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941441.889625:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941441.889628:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604280 00000400:00000200:3.0:1773941441.889632:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941441.889635:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xc35 [1] + 224 00000800:00000001:3.0:1773941441.889641:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941441.889647:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941441.889649:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941441.889651:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941441.889655:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941441.889659:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90276300 x1860112241148544/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941457 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941441.889667:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90276300 x1860112241148544/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941457 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941441.889692:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941441.889697:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941441.889700:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941441.889775:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941441.889782:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941441.889783:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941441.889786:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab058. 00000400:00000200:1.0:1773941441.889790:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6ad48 00000400:00000010:1.0:1773941441.889791:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6ad48. 00000100:00000001:1.0:1773941441.889794:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941441.889799:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90276300 x1860112241148544/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941457 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941441.889807:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90276300 x1860112241148544/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941457 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941441.889812:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941441.889813:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941441.889815:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941441.889818:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941441.889819:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941441.889821:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941441.889822:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941441.889826:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=20 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941441.889829:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941441.889831:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941441.889834:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941441.889836:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941441.889838:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941441.889840:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941441.889843:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941441.889854:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941441.889857:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941441.889858:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941441.889860:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941441.889861:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941441.889864:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90276300 x1860112241148544/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941457 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941441.889886:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941441.889887:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941441.889889:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941441.889891:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941441.889893:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941441.889894:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941441.889895:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941441.889896:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941441.889897:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941441.889898:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941441.889899:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941441.889901:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941441.889903:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941441.889905:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941441.889907:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 210 00000400:00000200:1.0:1773941441.889910:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941441.889914:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941441.889916:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941441.889918:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941441.889920:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941441.889922:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941441.889924:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941441.889925:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941441.889927:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941441.889929:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941441.889930:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941441.889936:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941441.889939:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941441.889940:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941441.889942:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941441.889943:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941441.889945:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941441.889947:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941441.889949:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941441.889951:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941441.889953:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941441.889955:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941441.889957:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941441.889958:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941441.889961:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63480. 00000020:00000040:1.0:1773941441.889963:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941441.889965:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941441.889966:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941441.889967:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941441.889969:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941441.889970:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941441.889974:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737000. 00000100:00000001:1.0:1773941441.889977:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941441.889981:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63480 x1860112241148672/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941441.889987:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941441.889989:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941441.889992:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000834a8cee] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63480 x1860112241148672/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941441.890009:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941441.890011:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941441.890012:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941441.890014:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941441.890020:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941441.890021:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941441.890022:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 1 seconds 00000100:00000001:1.0:1773941441.890024:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941441.890026:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941441.890031:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90276300 x1860112241148544/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941457 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941441.890039:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941441.890042:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941441.890049:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941441.890051:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941441.890056:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90276300 x1860112241148544/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941457 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941441.890066:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241148544:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941441.890071:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941441.890075:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941441.890076:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941441.890080:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941441.890081:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941441.890085:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941441.890085:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941441.890091:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63480 x1860112241148672/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941441.890094:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90276300 x1860112241148544/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941457 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941441.890112:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941441.890116:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941441.890117:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941441.890119:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:1.0:1773941441.890121:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e300. 02000000:00000001:2.0:1773941441.890122:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941441.890126:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941441.890126:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941441.890129:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:2.0:1773941441.890133:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241148672:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941441.890134:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941441.890138:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941441.890140:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941441.890141:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484200 refcount=4 00000020:00000001:1.0:1773941441.890143:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860122112 : -128249849429504 : ffff8b5b81484200) 00010000:00000001:1.0:1773941441.890147:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860122112 : -128249849429504 : ffff8b5b81484200) 00000100:00000001:2.0:1773941441.890151:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00010000:1.0:1773941441.890153:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1ca6 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941441.890154:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941441.890156:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941441.890157:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941441.890161:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce4c00. 02000000:00000001:2.0:1773941441.890165:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941441.890172:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941441.890173:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000010:2.0:1773941441.890175:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00010000:00010000:1.0:1773941441.890178:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1ca6 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941441.890181:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000200:2.0:1773941441.890186:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241148672, portal 25 00010000:00000001:1.0:1773941441.890187:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:2.0:1773941441.890192:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00010000:1.0:1773941441.890193:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1ca6 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941441.890194:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083008 : -128249588468608 : ffff8b5b90d63480) 00000100:00000040:2.0:1773941441.890203:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63480 x1860112241148672/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941441.890204:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941441.890208:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941441.890212:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1ca6 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941441.890215:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941441.890221:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941441.890224:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000010:2.0:1773941441.890225:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34dd0. 00010000:00000001:1.0:1773941441.890227:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941441.890229:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241148672, offset 0 00010000:00000001:1.0:1773941441.890231:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941441.890236:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941441.890236:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1ca6 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941441.890242:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941441.890247:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81484200 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:2.0:1773941441.890249:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 10000000:00000001:1.0:1773941441.890251:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941441.890254:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00010000:1.0:1773941441.890255:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1ca6 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941441.890260:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941441.890266:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941441.890266:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941441.890270:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941441.890271:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1773941441.890273:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941441.890277:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[34, 35] 00000020:00000040:1.0:1773941441.890278:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484200 refcount=5 00000020:00000001:1.0:1773941441.890283:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860122112 : -128249849429504 : ffff8b5b81484200) 00000400:00000200:2.0:1773941441.890285:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941441.890288:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860122112 : -128249849429504 : ffff8b5b81484200) 00000400:00000200:2.0:1773941441.890290:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [141:141:256:4294967295] 192.168.204.143@tcp LPNI seq info [36:36:8:4294967295] 00000400:00000200:2.0:1773941441.890296:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941441.890297:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941441.890301:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1ca6 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941441.890305:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 10000000:00000001:1.0:1773941441.890316:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:2.0:1773941441.890317:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 10000000:00010000:1.0:1773941441.890320:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1ca6 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941441.890329:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000800:00000010:2.0:1773941441.890330:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0a00. 10000000:00000040:1.0:1773941441.890332:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941441.890334:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941441.890338:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00010000:1.0:1773941441.890339:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941441.890342:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941441.890345:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941441.890347:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000800:00000200:2.0:1773941441.890348:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941441.890349:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000800:00000200:2.0:1773941441.890353:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0a00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941441.890354:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941441.890360:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81484200 with handle 0xa33a5be14eae1ca6 from hash 00010000:00000001:1.0:1773941441.890362:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941441.890363:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941441.890364:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941441.890365:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941441.890367:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941441.890368:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941441.890369:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941441.890371:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941441.890372:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941441.890373:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941441.890377:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941441.890380:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941441.890380:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1ca6 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941441.890384:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294882253 00010000:00000001:1.0:1773941441.890389:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941441.890392:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294882251 00010000:00000001:1.0:1773941441.890392:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941441.890395:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:2.0:1773941441.890398:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294882253 00010000:00000001:1.0:1773941441.890399:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:2.0:1773941441.890404:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294882251 00010000:00000001:1.0:1773941441.890408:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941441.890409:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941441.890410:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941441.890413:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941441.890414:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941441.890418:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1ca6 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941441.890419:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941441.890424:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:1.0:1773941441.890430:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962600 count: 0 00000100:00000001:2.0:1773941441.890433:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941441.890435:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941441.890437:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941441.890438:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941441.890440:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941441.890442:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941441.890443:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941441.890445:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941441.890446:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000010:3.0:1773941441.890451:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0a00. 00000100:00000001:1.0:1773941441.890452:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1773941441.890459:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:1.0:1773941441.890459:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90276300 x1860112241148544/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941457 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941441.890467:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:1.0:1773941441.890474:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000400:00000200:3.0:1773941441.890475:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34dd0 02000000:00000001:1.0:1773941441.890477:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000010:3.0:1773941441.890478:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34dd0. 02000000:00000010:1.0:1773941441.890480:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90263000. 00000100:00000001:3.0:1773941441.890482:0:2347:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1773941441.890485:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000200:3.0:1773941441.890488:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63480 x1860112241148672/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1773941441.890489:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941441.890492:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941441.890495:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941441.890498:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e3000. 02000000:00000001:1.0:1773941441.890501:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941441.890502:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941441.890503:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90276300. 00000100:00000001:1.0:1773941441.890506:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941441.890506:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941441.890508:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941441.890510:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941441.890511:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941441.890514:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0cc0. 00000020:00000010:1.0:1773941441.890517:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941441.890521:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941441.890524:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941441.890528:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941441.890531:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941441.890537:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000001:3.0:1773941441.890539:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941441.890543:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63480 x1860112241148672/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941451 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00040000:00000010:1.0:1773941441.890546:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941441.890552:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 00000100:00000001:3.0:1773941441.890556:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:1.0:1773941441.890558:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000100:00000001:3.0:1773941441.890561:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000001:1.0:1773941441.890562:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941441.890567:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941441.890571:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941441.890573:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941441.890576:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941441.890579:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941441.890583:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941441.890585:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941441.890587:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941441.890588:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0cc0. 10000000:01000000:1.0:1773941441.890590:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941441.890592:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941441.890594:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000800:00000001:3.0:1773941441.891724:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941441.891729:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941441.891730:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941441.891732:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941441.891735:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941441.891738:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941441.891742:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941441.891744:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941441.891746:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941441.891748:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604300 00000400:00000200:3.0:1773941441.891751:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941441.891754:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xc45 [1] + 0 00000800:00000001:3.0:1773941441.891757:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941441.891760:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941441.891762:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941441.891801:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941441.891804:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941441.891807:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63480 x1860112241148672/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941451 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941441.891814:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63480 x1860112241148672/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941451 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941441.891825:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941441.891829:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941441.891831:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941441.891850:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941441.891853:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941441.891856:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941441.891857:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941441.891858:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941441.891861:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941441.891862:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941441.891864:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941441.891878:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63480 x1860112241148672/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941451 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941441.891886:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63480 x1860112241148672/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941451 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941441.891891:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941441.891897:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941441.891898:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941441.891900:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941441.891901:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941441.891902:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941441.891904:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941441.891906:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=20 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941441.891909:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941441.891910:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941441.891912:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941441.891913:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941441.891915:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941441.891916:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941441.891919:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63480 x1860112241148672/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941451 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941441.891924:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941441.891925:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941441.891926:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941441.891928:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941441.891931:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941441.891933:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941441.891934:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941441.891935:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941441.891937:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941441.891938:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941441.891940:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941441.891942:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941441 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941441.891944:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941441.891949:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941441.891950:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941441.891952:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941441.891953:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941441.891954:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941441.891957:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941441.891958:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941441.891963:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941441.891965:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941441.891968:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941441.891970:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941441.892170:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941441.892174:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941441.892181:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63480 x1860112241148672/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941451 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941441.892190:0:12167:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:0.0:1773941441.892191:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241148672:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941441.892198:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941441.892201:0:12167:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:0.0:1773941441.892202:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1773941441.892207:0:12167:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:0.0:1773941441.892209:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63480 x1860112241148672/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941451 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941441.892210:0:12167:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941441.892213:0:12167:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941441.892217:0:12167:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941441.892221:0:12167:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:0.0:1773941441.892221:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941441.892225:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:2.0:1773941441.892226:0:12167:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941441.892229:0:12167:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:0.0:1773941441.892229:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce4c00. 00000001:00000001:2.0:1773941441.892233:0:12167:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941441.892235:0:12167:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941441.892235:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941441.892237:0:2351:0:(genops.c:1236:class_import_put()) Process entered 10000000:00000001:2.0:1773941441.892238:0:12167:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941441.892240:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941441.892241:0:12167:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:0.0:1773941441.892245:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:2.0:1773941441.892247:0:12167:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:0.0:1773941441.892248:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737000. 00000100:00000001:2.0:1773941441.892254:0:12167:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941441.892256:0:12167:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941441.892257:0:12167:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941441.892258:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:2.0:1773941441.892259:0:12167:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 02000000:00000001:0.0:1773941441.892262:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941441.892263:0:12167:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000010:0.0:1773941441.892265:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63480. 10000000:01000000:2.0:1773941441.892266:0:12167:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941441.892267:0:12167:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:0.0:1773941441.892275:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941441.892276:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941441.892278:0:12167:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:0.0:1773941441.892278:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941441.892281:0:12167:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941441.892282:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941441.892283:0:12167:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941441.892287:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:02000000:2.0:1773941441.892291:0:12167:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941441.892292:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941441.892296:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941441.892298:0:12167:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941441.892301:0:12167:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941441.892302:0:12167:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941441.892303:0:12167:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941441.892304:0:12167:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941441.894715:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941441.894717:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941441.894719:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941441.894721:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:1.1:1773941441.901752:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962600. 00010000:00000010:1.1:1773941441.901761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81484200. 00000020:00000001:0.0:1773941442.399903:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941442.399911:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941442.399916:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941442.463884:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (217) 10000000:00000001:1.0:1773941443.103836:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941443.103858:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941443.103863:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941443.103866:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941443.103868:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941443.103870:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941443.103872:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941443.103874:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941443.103878:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941443.103881:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941443.103882:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941443.103893:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60380. 00000020:00000040:1.0:1773941443.103898:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941443.103901:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941443.103904:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941443.103923:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941443.103931:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941443.103939:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941443.103945:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941443.103948:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941443.103956:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ecd00. 00010000:00000001:1.0:1773941443.103962:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941443.103966:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f000. 00000020:00000001:1.0:1773941443.103970:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941443.103973:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f000 with handle 0xa33a5be14eae1cad to hash 00000020:00000001:1.0:1773941443.103976:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941443.103977:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941443.103979:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941443.103983:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941443.103985:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941443.103989:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1cad lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941443.103998:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1cad lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941443.104007:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1cad lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941443.104012:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941443.104014:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941443.104021:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941443.104023:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941443.104026:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941443.104027:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121070464 : -128249588481152 : ffff8b5b90d60380) 00000100:00000001:1.0:1773941443.104033:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941443.104037:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941443.104039:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941443.104041:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941443.104057:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60380 x1860112241148800/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941443.104068:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941443.104070:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941443.104074:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941443.104076:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941443.104079:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941443.104084:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241148800:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941443.104089:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941443.104094:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941443.104096:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941443.104099:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941443.104104:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735400. 02000000:00000001:1.0:1773941443.104106:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941443.104114:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941443.104118:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941443.104126:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941443.104145:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241148800, portal 25 00000100:00000001:1.0:1773941443.104147:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941443.104148:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121070464 : -128249588481152 : ffff8b5b90d60380) 00000100:00000040:1.0:1773941443.104153:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60380 x1860112241148800/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941459 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941443.104159:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941443.104161:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941443.104164:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1000. 00000100:00000200:1.0:1773941443.104166:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241148800, offset 0 00000400:00000200:1.0:1773941443.104171:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941443.104178:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941443.104184:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941443.104188:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941443.104191:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941443.104195:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941443.104203:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941443.104205:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[36, 35] 00000400:00000200:1.0:1773941443.104208:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941443.104211:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [142:142:256:4294967295] 192.168.204.153@tcp LPNI seq info [37:37:8:4294967295] 00000400:00000200:1.0:1773941443.104215:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941443.104223:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941443.104227:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941443.104231:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b100. 00000800:00000200:1.0:1773941443.104237:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941443.104243:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941443.104247:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b100 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941443.104261:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941443.104264:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941443.104266:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294883467 00000100:00000001:1.0:1773941443.104272:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294882253 00000100:00000001:1.0:1773941443.104276:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294883467 00000100:00000001:1.0:1773941443.104279:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294882253 00000100:00000001:1.0:1773941443.104281:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941443.104283:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941443.104285:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941443.104286:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941443.104289:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941443.104291:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941443.104294:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941443.104441:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941443.104448:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b100. 00000400:00000200:3.0:1773941443.104455:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941443.104463:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941443.104471:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1000 00000400:00000010:3.0:1773941443.104473:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1000. 00000100:00000001:3.0:1773941443.104477:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941443.104484:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60380 x1860112241148800/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941459 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941443.104495:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941443.104498:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d60380 x1860112241148800/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941459 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941443.104504:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941443.104505:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941443.105375:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941443.105389:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941443.105392:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941443.105395:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941443.105402:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941443.105408:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941443.105417:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941443.105465:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941443.105473:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941443.105491:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604380 00000400:00000200:3.0:1773941443.105496:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941443.105505:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xc55 [1] + 224 00000800:00000001:3.0:1773941443.105529:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941443.105539:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941443.105542:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941443.105546:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941443.105552:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941443.105558:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60380 x1860112241148800/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941459 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941443.105570:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d60380 x1860112241148800/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941459 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941443.105592:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941443.105600:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941443.105604:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941443.105630:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941443.105636:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941443.105637:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941443.105640:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941443.105645:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941443.105647:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941443.105649:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941443.105661:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60380 x1860112241148800/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941459 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941443.105670:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60380 x1860112241148800/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941459 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941443.105675:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941443.105677:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941443.105679:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941443.105681:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941443.105682:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941443.105684:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941443.105686:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941443.105691:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=22 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941443.105693:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941443.105695:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941443.105698:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941443.105700:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941443.105720:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941443.105722:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941443.105726:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941443.105729:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941443.105731:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941443.105733:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941443.105734:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941443.105735:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941443.105738:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d60380 x1860112241148800/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941459 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941443.105743:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941443.105744:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941443.105745:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941443.105747:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941443.105749:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941443.105750:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941443.105755:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941443.105756:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941443.105757:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941443.105758:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941443.105759:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941443.105761:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941443.105762:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941443.105764:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941443.105766:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 212 00000400:00000200:1.0:1773941443.105770:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941443.105774:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941443.105777:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941443.105779:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941443.105781:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941443.105783:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941443.105786:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941443.105787:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941443.105789:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941443.105791:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941443.105792:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941443.105793:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941443.105795:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941443.105796:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941443.105798:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941443.105799:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941443.105801:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941443.105803:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941443.105805:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941443.105806:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941443.105809:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941443.105810:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941443.105812:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941443.105816:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941443.105819:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60000. 00000020:00000040:1.0:1773941443.105821:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941443.105823:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941443.105824:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941443.105826:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941443.105827:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941443.105829:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941443.105832:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734800. 00000100:00000001:1.0:1773941443.105835:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941443.105840:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60000 x1860112241148928/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941443.105860:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941443.105862:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941443.105866:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000b65cec4] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60000 x1860112241148928/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941443.105884:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941443.105887:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941443.105888:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941443.105890:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941443.105892:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941443.105893:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941443.105894:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941443.105896:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941443.105897:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941443.105900:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60380 x1860112241148800/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941459 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941443.105906:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941443.105907:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941443.105909:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941443.105911:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941443.105914:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60380 x1860112241148800/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941459 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941443.105929:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241148800:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941443.105933:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941443.105933:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941443.105937:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941443.105940:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941443.105940:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941443.105944:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941443.105945:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941443.105947:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60380 x1860112241148800/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941459 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941443.105955:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941443.105956:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60000 x1860112241148928/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941443.105957:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941443.105962:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941443.105964:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941443.105969:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941443.105972:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941443.105976:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1773941443.105976:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1773941443.105981:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=4 00000100:00000001:0.0:1773941443.105981:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941443.105985:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941443.105988:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 02000000:00000001:0.0:1773941443.105988:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941443.105993:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00010000:1.0:1773941443.105994:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1cad lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941443.106000:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941443.106003:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00100000:0.0:1773941443.106008:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241148928:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00010000:1.0:1773941443.106009:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1cad lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941443.106019:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941443.106021:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000001:0.0:1773941443.106024:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00010000:1.0:1773941443.106027:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1cad lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941443.106027:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941443.106032:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941443.106039:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000010:0.0:1773941443.106039:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0c00. 02000000:00000001:0.0:1773941443.106042:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941443.106046:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941443.106049:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:0.0:1773941443.106050:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00010000:1.0:1773941443.106054:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1cad lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941443.106056:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941443.106061:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241148928, portal 25 00000100:00000001:0.0:1773941443.106064:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941443.106066:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121069568 : -128249588482048 : ffff8b5b90d60000) 00010000:00000001:1.0:1773941443.106068:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941443.106072:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941443.106073:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60000 x1860112241148928/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941453 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941443.106076:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941443.106080:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1cad lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941443.106100:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f000 already unused, calling callback (ffffffffc1482c20) 00000100:00000001:0.0:1773941443.106100:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 10000000:00000001:1.0:1773941443.106104:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000040:0.0:1773941443.106105:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 10000000:00010000:1.0:1773941443.106107:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1cad lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941443.106108:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6cb28. 00000100:00000200:0.0:1773941443.106114:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241148928, offset 0 00010000:00000001:1.0:1773941443.106117:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941443.106119:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:0.0:1773941443.106120:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000020:00000001:1.0:1773941443.106122:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941443.106125:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f000 refcount=5 00000400:00000200:0.0:1773941443.106129:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:1.0:1773941443.106130:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00010000:00000001:1.0:1773941443.106135:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859118080 : -128249850433536 : ffff8b5b8138f000) 00000400:00000200:0.0:1773941443.106136:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:1.0:1773941443.106140:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:0.0:1773941443.106142:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:1.0:1773941443.106145:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1cad lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941443.106149:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941443.106155:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941443.106160:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000001:1.0:1773941443.106162:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941443.106167:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[36, 37] 10000000:00010000:1.0:1773941443.106168:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1cad lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941443.106178:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 10000000:01000000:1.0:1773941443.106180:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941443.106185:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:0.0:1773941443.106186:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [143:143:256:4294967295] 192.168.204.143@tcp LPNI seq info [38:38:8:4294967295] 10000000:00000001:1.0:1773941443.106187:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941443.106193:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941443.106195:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:0.0:1773941443.106196:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941443.106205:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:0.0:1773941443.106206:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941443.106209:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941443.106212:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941443.106215:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941443.106215:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000020:00000040:1.0:1773941443.106220:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f000 with handle 0xa33a5be14eae1cad from hash 00010000:00000001:1.0:1773941443.106222:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000800:00000010:0.0:1773941443.106222:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00010000:00000001:1.0:1773941443.106225:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941443.106227:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941443.106229:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000800:00000200:0.0:1773941443.106230:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941443.106231:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941443.106235:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941443.106238:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941443.106238:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941443.106242:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941443.106242:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 10000000:00000001:1.0:1773941443.106247:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941443.106248:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:1.0:1773941443.106251:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1cad lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941443.106256:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941443.106257:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941443.106257:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941443.106259:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941443.106260:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941443.106260:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941443.106261:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941443.106263:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941443.106264:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f000/0xa33a5be14eae1cad lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941443.106271:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ecd00 count: 0 00000400:00000200:0.0:1773941443.106276:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941443.106279:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294883469 00010000:00000001:1.0:1773941443.106283:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941443.106286:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941443.106287:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294883467 00000100:00000001:1.0:1773941443.106289:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941443.106294:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294883469 00000100:00000040:1.0:1773941443.106295:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60380 x1860112241148800/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941459 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941443.106298:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294883467 00000100:00000001:0.0:1773941443.106304:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941443.106307:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941443.106308:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941443.106310:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1773941443.106318:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941443.106321:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1773941443.106322:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735400. 02000000:00000001:1.0:1773941443.106326:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941443.106328:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941443.106328:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941443.106332:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000040:1.0:1773941443.106333:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941443.106335:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1773941443.106337:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941443.106338:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 02000000:00000010:1.0:1773941443.106341:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 00000100:00000001:0.0:1773941443.106341:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941443.106343:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941443.106348:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000800:00000200:3.0:1773941443.106350:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 02000000:00000001:1.0:1773941443.106353:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000800:00000010:3.0:1773941443.106356:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000100:00000010:1.0:1773941443.106357:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60380. 00000400:00000200:3.0:1773941443.106364:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1773941443.106365:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941443.106372:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941443.106378:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 10000000:00000001:1.0:1773941443.106380:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:3.0:1773941443.106384:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6cb28 10000000:01000000:1.0:1773941443.106386:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941443.106392:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941443.106396:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0cc0. 00000020:00000010:1.0:1773941443.106401:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941443.106405:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941443.106408:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941443.106412:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941443.106415:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941443.106418:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941443.106422:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941443.106426:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 10000000:00000040:1.0:1773941443.106430:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000400:00000010:3.0:1773941443.106431:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6cb28. 10000000:00000001:1.0:1773941443.106432:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941443.106441:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000100:00000001:3.0:1773941443.106442:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000040:00000010:1.0:1773941443.106447:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000100:00000200:3.0:1773941443.106454:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60000 x1860112241148928/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941453 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941443.106454:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941443.106468:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 00000100:00000001:3.0:1773941443.106478:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 80000000:00000010:1.0:1773941443.106479:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00000100:00000040:3.0:1773941443.106488:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60000 x1860112241148928/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941453 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00040000:00000010:1.0:1773941443.106490:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941443.106500:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000100:00000001:3.0:1773941443.106512:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941443.106512:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 00000100:00000001:3.0:1773941443.106524:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000010:1.0:1773941443.106527:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0cc0. 10000000:01000000:1.0:1773941443.106535:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941443.106544:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941443.106548:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941443.106550:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941443.106552:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941443.106553:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941443.106556:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941443.108190:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941443.108198:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941443.108201:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941443.108213:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941443.108219:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941443.108224:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941443.108230:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941443.108234:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941443.108298:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941443.108303:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604400 00000400:00000200:3.0:1773941443.108307:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941443.108312:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xc5d [1] + 0 00000800:00000001:3.0:1773941443.108318:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941443.108326:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941443.108329:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941443.108333:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941443.108337:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941443.108355:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60000 x1860112241148928/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941453 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941443.108368:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60000 x1860112241148928/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941453 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941443.108387:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941443.108392:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941443.108396:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941443.108426:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941443.108429:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941443.108432:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941443.108434:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941443.108436:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941443.108440:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941443.108441:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941443.108444:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941443.108449:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60000 x1860112241148928/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941453 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941443.108458:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60000 x1860112241148928/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941453 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941443.108463:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941443.108464:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941443.108466:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941443.108467:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941443.108468:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941443.108470:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941443.108471:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941443.108474:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=22 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941443.108476:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941443.108478:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941443.108479:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941443.108481:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941443.108482:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941443.108484:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941443.108486:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60000 x1860112241148928/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941453 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941443.108503:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941443.108504:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941443.108505:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941443.108507:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941443.108511:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941443.108512:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941443.108513:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941443.108514:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941443.108516:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941443.108517:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941443.108519:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941443.108522:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941443 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941443.108524:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941443.108528:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941443.108530:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941443.108531:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941443.108533:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941443.108534:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941443.108536:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941443.108538:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941443.108540:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941443.108542:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941443.108544:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941443.108546:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941443.108773:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941443.108782:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941443.108786:0:12168:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941443.108789:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60000 x1860112241148928/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941453 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941443.108795:0:12168:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941443.108801:0:12168:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:2.0:1773941443.108803:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241148928:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941443.108813:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941443.108815:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941443.108818:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60000 x1860112241148928/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941453 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941443.108824:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941443.108826:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1773941443.108826:0:12168:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941443.108830:0:12168:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000010:2.0:1773941443.108831:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0c00. 00000001:00000001:0.0:1773941443.108833:0:12168:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941443.108835:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941443.108836:0:12168:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:2.0:1773941443.108837:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941443.108839:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941443.108841:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941443.108842:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734800. 00000001:00000001:0.0:1773941443.108860:0:12168:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941443.108861:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941443.108864:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:0.0:1773941443.108864:0:12168:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941443.108868:0:12168:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000010:2.0:1773941443.108869:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60000. 00000001:00000001:0.0:1773941443.108872:0:12168:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941443.108873:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 10000000:00000001:0.0:1773941443.108875:0:12168:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941443.108876:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941443.108879:0:12168:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941443.108880:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941443.108884:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1773941443.108884:0:12168:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941443.108887:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941443.108889:0:12168:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941443.108890:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941443.108892:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941443.108892:0:12168:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941443.108895:0:12168:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941443.108898:0:12168:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941443.108899:0:12168:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941443.108900:0:12168:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941443.108902:0:12168:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941443.108916:0:12168:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941443.108917:0:12168:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941443.108919:0:12168:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941443.108923:0:12168:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941443.108926:0:12168:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941443.108929:0:12168:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941443.108931:0:12168:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941443.108934:0:12168:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941443.108936:0:12168:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941443.111753:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ecd00. 00010000:00000010:1.1:1773941443.111762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f000. 00000020:00000001:3.0:1773941443.423891:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941443.423897:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941443.423902:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941444.447867:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 00000020:00000001:0.0:1773941444.447874:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 10000000:00000040:3.0:1773941444.447878:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 00000020:00000001:0.0:1773941444.447882:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941444.447887:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941444.447889:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941444.447891:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941444.447894:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941444.447896:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941444.447899:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941444.447901:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941444.447903:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941444.447907:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941444.447930:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941444.447939:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275500. 00000020:00000040:3.0:1773941444.447943:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941444.447949:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941444.447952:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941444.447959:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941444.447967:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2600. 00000100:00000001:3.0:1773941444.447974:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941444.447979:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941444.447982:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941444.447990:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962200. 00010000:00000001:3.0:1773941444.447996:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941444.448000:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485200. 00000020:00000001:3.0:1773941444.448004:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941444.448006:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485200 with handle 0xa33a5be14eae1cb4 to hash 00000020:00000001:3.0:1773941444.448008:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941444.448009:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00010000:00000001:3.0:1773941444.448012:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00010000:00000001:3.0:1773941444.448016:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941444.448019:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941444.448022:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1cb4 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941444.448031:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1cb4 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941444.448043:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1cb4 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941444.448051:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941444.448053:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941444.448062:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e300. 00000100:00000001:3.0:1773941444.448065:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387400448 : -128249322151168 : ffff8b5ba0b5e300) 00000100:00000001:3.0:1773941444.448068:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941444.448075:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109622528 : -128249599929088 : ffff8b5b90275500) 00000100:00000001:3.0:1773941444.448082:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941444.448088:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941444.448091:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941444.448093:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941444.448107:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275500 x1860112241149056/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941444.448118:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941444.448120:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941444.448124:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941444.448126:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941444.448128:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941444.448132:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241149056:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941444.448136:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941444.448142:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941444.448144:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941444.448147:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941444.448151:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941444.448152:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941444.448160:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941444.448164:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abe18. 00000400:00000010:3.0:1773941444.448170:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a4c8. 00000100:00000200:3.0:1773941444.448177:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241149056, portal 25 00000100:00000001:3.0:1773941444.448178:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941444.448180:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109622528 : -128249599929088 : ffff8b5b90275500) 00000100:00000040:3.0:1773941444.448184:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275500 x1860112241149056/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941460 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941444.448190:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941444.448192:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941444.448194:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6ad48. 00000100:00000200:3.0:1773941444.448197:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241149056, offset 0 00000400:00000200:3.0:1773941444.448202:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941444.448211:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941444.448223:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941444.448229:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941444.448234:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941444.448237:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941444.448240:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941444.448242:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[38, 37] 00000400:00000200:3.0:1773941444.448245:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941444.448248:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [144:144:256:4294967295] 192.168.204.153@tcp LPNI seq info [39:39:8:4294967295] 00000400:00000200:3.0:1773941444.448252:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941444.448262:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941444.448267:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941444.448271:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69600. 00000800:00000200:3.0:1773941444.448278:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941444.448285:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941444.448290:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941444.448311:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941444.448314:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941444.448316:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294884811 00000100:00000001:3.0:1773941444.448323:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294883469 00000100:00000001:3.0:1773941444.448330:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294884811 00000100:00000001:3.0:1773941444.448335:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294883469 00000100:00000001:3.0:1773941444.448340:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941444.448342:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941444.448344:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941444.448346:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941444.448348:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 11 seconds 00000100:00000001:3.0:1773941444.448351:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941444.448354:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941444.448426:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941444.448431:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69600. 00000400:00000200:2.0:1773941444.448437:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941444.448445:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941444.448451:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6ad48 00000400:00000010:2.0:1773941444.448455:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6ad48. 00000100:00000001:2.0:1773941444.448461:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941444.448469:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275500 x1860112241149056/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941460 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941444.448496:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941444.448501:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275500 x1860112241149056/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941460 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941444.448510:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941444.448512:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941444.449516:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941444.449530:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941444.449532:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941444.449540:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941444.449581:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941444.449586:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941444.449592:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941444.449595:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941444.449600:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941444.449603:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604480 00000400:00000200:3.0:1773941444.449607:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941444.449611:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xc65 [1] + 224 00000800:00000001:3.0:1773941444.449617:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941444.449625:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941444.449627:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941444.449631:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941444.449636:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941444.449641:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275500 x1860112241149056/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941460 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941444.449649:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275500 x1860112241149056/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941460 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941444.449671:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941444.449680:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941444.449684:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941444.449737:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941444.449745:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941444.449747:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941444.449751:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abe18. 00000400:00000200:1.0:1773941444.449755:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a4c8 00000400:00000010:1.0:1773941444.449756:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a4c8. 00000100:00000001:1.0:1773941444.449759:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941444.449764:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275500 x1860112241149056/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941460 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941444.449772:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275500 x1860112241149056/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941460 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941444.449778:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941444.449779:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941444.449782:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941444.449784:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941444.449786:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941444.449787:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941444.449790:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941444.449795:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=23 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941444.449798:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941444.449800:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941444.449803:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941444.449806:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941444.449808:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941444.449810:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941444.449813:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941444.449816:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941444.449842:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941444.449844:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941444.449846:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941444.449847:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941444.449850:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275500 x1860112241149056/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941460 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941444.449856:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941444.449857:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941444.449859:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941444.449862:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941444.449864:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941444.449865:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941444.449866:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941444.449867:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941444.449868:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941444.449869:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941444.449870:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941444.449872:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941444.449873:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941444.449875:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941444.449877:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 213 00000400:00000200:1.0:1773941444.449883:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941444.449887:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941444.449890:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941444.449892:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941444.449894:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941444.449895:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941444.449897:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941444.449898:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941444.449901:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941444.449903:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941444.449904:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941444.449906:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941444.449911:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941444.449913:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941444.449914:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941444.449915:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941444.449917:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941444.449919:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941444.449921:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941444.449923:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941444.449925:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941444.449927:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941444.449929:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941444.449930:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941444.449934:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60380. 00000020:00000040:1.0:1773941444.449936:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941444.449939:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941444.449940:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941444.449942:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941444.449944:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941444.449948:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941444.449953:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737c00. 00000100:00000001:1.0:1773941444.449959:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941444.449965:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60380 x1860112241149184/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941444.449974:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941444.449978:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941444.449983:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077e2ee70] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60380 x1860112241149184/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941444.450005:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941444.450008:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941444.450010:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941444.450013:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941444.450016:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941444.450023:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941444.450027:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 1 seconds 00000100:00000001:1.0:1773941444.450030:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941444.450032:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941444.450038:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1773941444.450039:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275500 x1860112241149056/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941460 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941444.450044:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941444.450047:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941444.450056:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60380 x1860112241149184/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941444.450060:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941444.450064:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941444.450069:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941444.450072:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941444.450075:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1773941444.450079:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275500 x1860112241149056/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941460 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941444.450080:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941444.450087:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:1.0:1773941444.450090:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241149056:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941444.450091:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941444.450097:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941444.450099:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941444.450103:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:2.0:1773941444.450105:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241149184:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941444.450107:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941444.450112:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941444.450114:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941444.450118:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1773941444.450119:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275500 x1860112241149056/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941460 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941444.450123:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941444.450128:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941444.450135:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce6c00. 00000100:00000001:1.0:1773941444.450135:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941444.450142:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941444.450142:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e300. 00000100:00000001:1.0:1773941444.450149:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000400:00000200:2.0:1773941444.450152:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941444.450152:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000010:2.0:1773941444.450159:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00010000:00000001:1.0:1773941444.450159:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941444.450162:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941444.450166:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:2.0:1773941444.450167:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000020:00000040:1.0:1773941444.450169:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485200 refcount=4 00000020:00000001:1.0:1773941444.450172:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00010000:00000001:1.0:1773941444.450175:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00000100:00000200:2.0:1773941444.450176:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241149184, portal 25 00010000:00010000:1.0:1773941444.450179:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1cb4 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941444.450181:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941444.450183:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121070464 : -128249588481152 : ffff8b5b90d60380) 00000100:00000040:2.0:1773941444.450192:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60380 x1860112241149184/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941454 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941444.450195:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941444.450202:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1cb4 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941444.450209:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941444.450212:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941444.450215:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1cb4 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941444.450222:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000040:2.0:1773941444.450224:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941444.450225:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941444.450228:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1cb4 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941444.450229:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34cc0. 00000100:00000200:2.0:1773941444.450236:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241149184, offset 0 00010000:00000001:1.0:1773941444.450236:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941444.450238:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941444.450243:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941444.450246:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1cb4 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941444.450249:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941444.450255:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485200 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941444.450258:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941444.450260:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1cb4 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941444.450261:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941444.450268:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941444.450270:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941444.450272:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000020:00000001:1.0:1773941444.450273:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941444.450276:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485200 refcount=5 00000020:00000001:1.0:1773941444.450280:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00010000:00000001:1.0:1773941444.450282:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00000400:00000200:2.0:1773941444.450283:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941444.450289:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941444.450290:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941444.450294:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1cb4 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941444.450295:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941444.450302:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000001:1.0:1773941444.450305:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941444.450310:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[38, 39] 10000000:00010000:1.0:1773941444.450310:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1cb4 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941444.450322:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 10000000:01000000:1.0:1773941444.450324:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941444.450327:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:2.0:1773941444.450330:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [145:145:256:4294967295] 192.168.204.143@tcp LPNI seq info [40:40:8:4294967295] 10000000:00000001:1.0:1773941444.450330:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941444.450335:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941444.450338:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941444.450342:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941444.450342:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941444.450345:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941444.450348:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941444.450350:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941444.450355:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485200 with handle 0xa33a5be14eae1cb4 from hash 00000400:00000200:2.0:1773941444.450357:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941444.450358:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941444.450361:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941444.450364:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941444.450366:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000800:00000200:2.0:1773941444.450367:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941444.450368:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941444.450371:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000010:2.0:1773941444.450373:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0a00. 00010000:00000001:1.0:1773941444.450373:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941444.450376:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941444.450378:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941444.450380:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941444.450383:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00010000:1.0:1773941444.450384:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1cb4 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941444.450390:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941444.450390:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941444.450392:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941444.450394:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000800:00000200:2.0:1773941444.450396:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941444.450401:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0a00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941444.450401:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941444.450403:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941444.450405:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941444.450409:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1cb4 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941444.450416:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962200 count: 0 00000100:00000001:2.0:1773941444.450425:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941444.450426:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941444.450429:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941444.450432:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1773941444.450435:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000040:1.0:1773941444.450438:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275500 x1860112241149056/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941460 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941444.450440:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294884813 00000100:00000001:1.0:1773941444.450450:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941444.450451:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294884811 02000000:00000001:1.0:1773941444.450452:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941444.450456:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 00000100:00000001:2.0:1773941444.450459:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294884813 02000000:00000001:1.0:1773941444.450460:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941444.450464:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941444.450466:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941444.450467:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294884811 00000020:00000001:1.0:1773941444.450468:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941444.450471:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2600. 00000100:00000001:2.0:1773941444.450473:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941444.450476:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1773941444.450479:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941444.450479:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941444.450481:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275500. 00000100:00000001:2.0:1773941444.450484:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941444.450485:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941444.450489:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941444.450490:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941444.450491:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941444.450493:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941444.450495:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941444.450498:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0cc0. 00000020:00000010:1.0:1773941444.450502:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000800:00000200:3.0:1773941444.450506:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941444.450506:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:1.0:1773941444.450507:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000100:00000001:2.0:1773941444.450510:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000010:3.0:1773941444.450513:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0a00. 00000040:00000010:1.0:1773941444.450513:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000100:00000001:2.0:1773941444.450514:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941444.450519:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1773941444.450520:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:1.0:1773941444.450521:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000100:00000001:2.0:1773941444.450523:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941444.450528:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 00000400:00000200:3.0:1773941444.450529:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941444.450530:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 80000000:00000010:1.0:1773941444.450534:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000400:00000200:3.0:1773941444.450536:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34cc0 00000400:00000010:3.0:1773941444.450539:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34cc0. 00000100:00000001:3.0:1773941444.450543:0:2347:0:(events.c:34:request_out_callback()) Process entered 00040000:00000010:1.0:1773941444.450546:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 00000100:00000200:3.0:1773941444.450549:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60380 x1860112241149184/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941454 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 20000000:00000010:1.0:1773941444.450551:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941444.450558:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000100:00000001:3.0:1773941444.450561:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00000001:1.0:1773941444.450561:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941444.450567:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941444.450570:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941444.450572:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941444.450574:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941444.450576:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941444.450580:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941444.450582:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941444.450584:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941444.450585:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0cc0. 10000000:01000000:1.0:1773941444.450587:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941444.450589:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941444.450591:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941444.450593:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941444.450595:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941444.450597:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941444.450599:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000040:3.0:1773941444.450624:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60380 x1860112241149184/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941454 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941444.450636:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941444.450638:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941444.452171:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941444.452181:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941444.452183:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941444.452185:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941444.452189:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941444.452193:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941444.452198:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941444.452239:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941444.452246:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941444.452251:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604500 00000400:00000200:3.0:1773941444.452255:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941444.452261:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xc75 [1] + 0 00000800:00000001:3.0:1773941444.452268:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941444.452275:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941444.452278:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941444.452281:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941444.452286:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941444.452291:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60380 x1860112241149184/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941454 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941444.452303:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60380 x1860112241149184/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941454 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941444.452318:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941444.452326:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941444.452329:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941444.452357:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941444.452360:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941444.452365:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941444.452366:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941444.452369:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941444.452373:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941444.452375:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941444.452378:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941444.452384:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60380 x1860112241149184/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941454 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941444.452396:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60380 x1860112241149184/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941454 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941444.452405:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941444.452406:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941444.452408:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941444.452410:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941444.452412:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941444.452414:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941444.452416:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941444.452420:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=23 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941444.452424:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941444.452426:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941444.452428:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941444.452431:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941444.452433:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941444.452436:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941444.452441:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60380 x1860112241149184/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941454 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941444.452449:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941444.452451:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941444.452459:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941444.452462:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941444.452468:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941444.452470:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941444.452472:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941444.452474:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941444.452476:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941444.452479:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941444.452481:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941444.452485:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941444 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941444.452488:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941444.452494:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941444.452497:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941444.452499:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941444.452501:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941444.452503:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941444.452507:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941444.452510:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941444.452512:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941444.452516:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941444.452519:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941444.452521:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941444.452771:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941444.452775:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941444.452782:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60380 x1860112241149184/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941454 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941444.452785:0:12169:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941444.452795:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241149184:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00080000:1.0:1773941444.452796:0:12169:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:1.0:1773941444.452802:0:12169:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941444.452805:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941444.452806:0:12169:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941444.452808:0:12169:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941444.452812:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000001:00000001:1.0:1773941444.452812:0:12169:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941444.452816:0:12169:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000040:2.0:1773941444.452835:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60380 x1860112241149184/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941454 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:1.0:1773941444.452837:0:12169:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941444.452840:0:12169:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941444.452841:0:12169:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941444.452842:0:12169:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941444.452845:0:12169:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941444.452848:0:12169:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941444.452853:0:12169:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941444.452855:0:12169:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941444.452865:0:12169:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941444.452868:0:12169:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941444.452872:0:12169:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941444.452876:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1773941444.452876:0:12169:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941444.452880:0:12169:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 02000000:00000001:2.0:1773941444.452881:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 10000000:00000040:1.0:1773941444.452882:0:12169:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 02000000:00000010:2.0:1773941444.452886:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce6c00. 02000000:00000001:2.0:1773941444.452892:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941444.452894:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941444.452896:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941444.452898:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941444.452900:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737c00. 00000100:00080000:1.0:1773941444.452904:0:12169:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 02000000:00000001:2.0:1773941444.452905:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000001:1.0:1773941444.452908:0:12169:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941444.452909:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941444.452911:0:12169:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000010:2.0:1773941444.452913:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60380. 00000100:02000000:1.0:1773941444.452917:0:12169:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941444.452922:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941444.452922:0:12169:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941444.452927:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1773941444.452927:0:12169:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941444.452930:0:12169:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941444.452932:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1773941444.452933:0:12169:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941444.452935:0:12169:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941444.452940:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941444.452944:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941444.452947:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941444.452949:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941444.455760:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962200. 00010000:00000010:1.1:1773941444.455769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485200. 00000020:00000001:3.0:1773941445.471896:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941445.471902:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941445.471905:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941445.727790:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941445.727809:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941445.727831:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941445.727833:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941445.727834:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941445.727835:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941445.727837:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941445.727839:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941445.727841:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941445.727843:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941445.727844:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941445.727853:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275880. 00000020:00000040:3.0:1773941445.727856:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941445.727860:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941445.727863:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941445.727868:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941445.727875:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2a00. 00000100:00000001:3.0:1773941445.727880:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941445.727884:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941445.727886:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941445.727893:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962300. 00010000:00000001:3.0:1773941445.727898:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941445.727901:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81484600. 00000020:00000001:3.0:1773941445.727904:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941445.727906:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81484600 with handle 0xa33a5be14eae1cbb to hash 00000020:00000001:3.0:1773941445.727908:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941445.727909:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:3.0:1773941445.727911:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:3.0:1773941445.727914:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941445.727917:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941445.727920:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1cbb lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941445.727928:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1cbb lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941445.727937:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1cbb lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941445.727945:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941445.727946:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941445.727951:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941445.727953:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941445.727955:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941445.727956:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109623424 : -128249599928192 : ffff8b5b90275880) 00000100:00000001:3.0:1773941445.727960:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941445.727963:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941445.727964:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941445.727966:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941445.727974:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275880 x1860112241149312/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941445.727980:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941445.727982:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941445.727985:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941445.727987:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941445.727988:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941445.727990:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241149312:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941445.727993:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941445.727997:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941445.728000:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941445.728001:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941445.728005:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90262000. 02000000:00000001:3.0:1773941445.728007:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941445.728013:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941445.728016:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abaa8. 00000400:00000010:3.0:1773941445.728021:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6abb0. 00000100:00000200:3.0:1773941445.728027:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241149312, portal 25 00000100:00000001:3.0:1773941445.728028:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941445.728029:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109623424 : -128249599928192 : ffff8b5b90275880) 00000100:00000040:3.0:1773941445.728033:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275880 x1860112241149312/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941461 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941445.728060:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941445.728064:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941445.728068:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a220. 00000100:00000200:3.0:1773941445.728071:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241149312, offset 0 00000400:00000200:3.0:1773941445.728079:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941445.728088:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941445.728096:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941445.728101:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941445.728106:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941445.728110:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941445.728113:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941445.728115:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[40, 39] 00000400:00000200:3.0:1773941445.728118:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941445.728122:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [146:146:256:4294967295] 192.168.204.153@tcp LPNI seq info [41:41:8:4294967295] 00000400:00000200:3.0:1773941445.728132:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941445.728140:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941445.728145:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941445.728150:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69c00. 00000800:00000200:3.0:1773941445.728155:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941445.728162:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941445.728165:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69c00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941445.728180:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941445.728183:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941445.728185:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294886091 00000100:00000001:3.0:1773941445.728190:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294884813 00000100:00000001:3.0:1773941445.728195:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294886091 00000100:00000001:3.0:1773941445.728198:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294884813 00000100:00000001:3.0:1773941445.728201:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941445.728203:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941445.728209:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941445.728211:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941445.728214:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941445.728216:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941445.728218:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941445.728291:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941445.728296:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69c00. 00000400:00000200:2.0:1773941445.728300:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941445.728305:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941445.728310:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a220 00000400:00000010:2.0:1773941445.728311:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a220. 00000100:00000001:2.0:1773941445.728315:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941445.728320:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275880 x1860112241149312/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941461 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941445.728328:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941445.728330:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275880 x1860112241149312/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941461 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941445.728336:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941445.728337:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941445.729641:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941445.729651:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941445.729653:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941445.729654:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941445.729658:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941445.729662:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941445.729666:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941445.729669:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941445.729672:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941445.729675:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604580 00000400:00000200:3.0:1773941445.729678:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941445.729727:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xc85 [1] + 224 00000800:00000001:3.0:1773941445.729734:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941445.729740:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941445.729742:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941445.729745:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941445.729748:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941445.729751:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275880 x1860112241149312/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941461 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941445.729758:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275880 x1860112241149312/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941461 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941445.729772:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941445.729777:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941445.729779:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941445.729816:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941445.729822:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941445.729823:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941445.729826:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abaa8. 00000400:00000200:1.0:1773941445.729829:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6abb0 00000400:00000010:1.0:1773941445.729830:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6abb0. 00000100:00000001:1.0:1773941445.729832:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941445.729837:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275880 x1860112241149312/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941461 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941445.729845:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275880 x1860112241149312/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941461 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941445.729850:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941445.729851:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941445.729853:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941445.729855:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941445.729857:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941445.729859:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941445.729861:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941445.729865:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=24 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941445.729868:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941445.729877:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941445.729880:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941445.729882:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941445.729885:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941445.729887:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941445.729890:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941445.729893:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941445.729896:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941445.729897:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941445.729899:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941445.729901:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941445.729905:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275880 x1860112241149312/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941461 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941445.729912:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941445.729914:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941445.729916:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941445.729918:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941445.729921:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941445.729923:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941445.729925:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941445.729926:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941445.729927:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941445.729929:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941445.729931:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941445.729933:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941445.729935:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941445.729939:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941445.729941:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 214 00000400:00000200:1.0:1773941445.729946:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941445.729952:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941445.729956:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941445.729959:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941445.729968:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941445.729970:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941445.729973:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941445.729975:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941445.729978:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941445.729980:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941445.729983:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941445.729985:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941445.729989:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941445.729991:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941445.729993:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941445.729995:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941445.729998:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941445.730001:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941445.730004:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941445.730006:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941445.730010:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941445.730012:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941445.730015:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941445.730017:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941445.730021:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60700. 00000020:00000040:1.0:1773941445.730024:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941445.730028:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941445.730029:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941445.730032:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941445.730033:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941445.730036:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941445.730041:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0735c00. 00000100:00000001:1.0:1773941445.730045:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941445.730051:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60700 x1860112241149440/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941445.730060:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941445.730069:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941445.730075:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006bd2135a] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60700 x1860112241149440/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941445.730097:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941445.730100:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941445.730102:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941445.730105:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941445.730108:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941445.730110:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941445.730113:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941445.730116:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941445.730118:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941445.730121:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1773941445.730125:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275880 x1860112241149312/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941461 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941445.730126:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941445.730136:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941445.730138:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:2.0:1773941445.730143:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60700 x1860112241149440/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941445.730144:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941445.730151:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941445.730155:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941445.730156:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941445.730159:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941445.730162:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941445.730165:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1773941445.730165:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275880 x1860112241149312/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941461 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941445.730167:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941445.730172:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241149440:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941445.730179:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941445.730182:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941445.730184:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941445.730188:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941445.730191:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce6000. 00000100:00100000:1.0:1773941445.730192:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241149312:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941445.730194:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941445.730200:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941445.730202:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1773941445.730204:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000100:00000001:1.0:1773941445.730207:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000010:2.0:1773941445.730209:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000001:1.0:1773941445.730213:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000200:2.0:1773941445.730214:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241149440, portal 25 00000100:00000001:2.0:1773941445.730218:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941445.730218:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941445.730219:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121071360 : -128249588480256 : ffff8b5b90d60700) 00000100:00000040:2.0:1773941445.730229:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60700 x1860112241149440/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941455 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941445.730230:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275880 x1860112241149312/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941461 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941445.730245:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941445.730250:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941445.730254:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34aa0. 00000100:00000001:1.0:1773941445.730255:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941445.730257:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241149440, offset 0 00000100:00000010:1.0:1773941445.730260:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000400:00000200:2.0:1773941445.730263:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000100:00000001:1.0:1773941445.730266:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000400:00000200:2.0:1773941445.730268:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941445.730272:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941445.730276:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1773941445.730277:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941445.730280:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941445.730283:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000200:2.0:1773941445.730284:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941445.730288:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941445.730290:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941445.730294:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000020:00000040:1.0:1773941445.730295:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484600 refcount=4 00000400:00000200:2.0:1773941445.730300:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[40, 41] 00000020:00000001:1.0:1773941445.730302:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00000400:00000200:2.0:1773941445.730306:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941445.730309:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00000400:00000200:2.0:1773941445.730310:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [147:147:256:4294967295] 192.168.204.143@tcp LPNI seq info [42:42:8:4294967295] 00000400:00000200:2.0:1773941445.730316:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941445.730317:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1cbb lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941445.730326:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941445.730334:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941445.730336:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000800:00000010:2.0:1773941445.730338:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0c00. 00010000:00010000:1.0:1773941445.730345:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1cbb lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941445.730346:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941445.730355:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941445.730359:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0c00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941445.730361:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941445.730366:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1cbb lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941445.730373:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941445.730375:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941445.730378:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941445.730379:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941445.730383:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294886093 00010000:00010000:1.0:1773941445.730384:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1cbb lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941445.730387:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294886091 00000100:00000001:2.0:1773941445.730393:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294886093 00010000:00000001:1.0:1773941445.730396:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:2.0:1773941445.730398:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294886091 00010000:00000001:1.0:1773941445.730400:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941445.730402:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941445.730405:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941445.730406:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000001:2.0:1773941445.730408:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941445.730412:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:1.0:1773941445.730412:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1cbb lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941445.730420:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941445.730422:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00010000:1.0:1773941445.730423:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81484600 already unused, calling callback (ffffffffc1482c20) 00000100:00000001:2.0:1773941445.730425:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941445.730428:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000001:1.0:1773941445.730429:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:2.0:1773941445.730430:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941445.730431:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941445.730433:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00010000:1.0:1773941445.730434:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1cbb lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:3.0:1773941445.730436:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0c00. 00000400:00000200:3.0:1773941445.730441:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941445.730449:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00010000:00000001:1.0:1773941445.730454:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941445.730459:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:3.0:1773941445.730460:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34aa0 00000020:00000001:1.0:1773941445.730461:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:3.0:1773941445.730463:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34aa0. 00000020:00000040:1.0:1773941445.730465:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484600 refcount=5 00000100:00000001:3.0:1773941445.730467:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1773941445.730471:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00000100:00000200:3.0:1773941445.730472:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60700 x1860112241149440/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941455 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941445.730479:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:1.0:1773941445.730483:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941445.730486:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1cbb lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:1.0:1773941445.730493:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941445.730496:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1cbb lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941445.730503:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941445.730506:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941445.730508:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941445.730511:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941445.730515:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:3.0:1773941445.730517:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000001:1.0:1773941445.730520:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000040:3.0:1773941445.730522:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60700 x1860112241149440/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941455 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941445.730524:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941445.730529:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941445.730532:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941445.730534:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941445.730535:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:1.0:1773941445.730541:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81484600 with handle 0xa33a5be14eae1cbb from hash 00010000:00000001:1.0:1773941445.730545:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941445.730548:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941445.730550:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941445.730551:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941445.730553:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941445.730555:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941445.730557:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941445.730558:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941445.730560:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941445.730568:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941445.730573:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1cbb lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941445.730579:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941445.730580:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941445.730582:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941445.730583:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941445.730585:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941445.730586:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941445.730588:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941445.730590:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1cbb lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941445.730598:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962300 count: 0 00010000:00000001:1.0:1773941445.730608:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941445.730610:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941445.730613:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941445.730619:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275880 x1860112241149312/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941461 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941445.730627:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941445.730629:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941445.730631:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90262000. 02000000:00000001:1.0:1773941445.730635:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941445.730637:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941445.730639:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941445.730641:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941445.730644:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2a00. 02000000:00000001:1.0:1773941445.730647:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941445.730649:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941445.730651:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275880. 00000100:00000001:1.0:1773941445.730654:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941445.730656:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941445.730658:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941445.730660:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941445.730663:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941445.730671:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0cc0. 00000020:00000010:1.0:1773941445.730675:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941445.730681:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941445.730685:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941445.730690:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941445.730694:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941445.730699:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941445.730749:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941445.730755:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 10000000:00000040:1.0:1773941445.730760:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941445.730762:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941445.730768:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941445.730772:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941445.730776:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941445.730782:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941445.730787:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941445.730792:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941445.730812:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941445.730815:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941445.730818:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0cc0. 10000000:01000000:1.0:1773941445.730821:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941445.730824:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941445.730827:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941445.730829:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941445.730831:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941445.730833:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941445.730835:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941445.731687:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941445.731693:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941445.731695:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941445.731697:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941445.731700:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941445.731725:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941445.731792:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941445.731807:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941445.731811:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941445.731815:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604600 00000400:00000200:3.0:1773941445.731820:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941445.731824:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xc95 [1] + 0 00000800:00000001:3.0:1773941445.731829:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941445.731837:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941445.731839:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941445.731843:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941445.731847:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941445.731852:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60700 x1860112241149440/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941455 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941445.731863:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60700 x1860112241149440/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941455 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941445.731880:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941445.731891:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941445.731894:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941445.731918:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941445.731921:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941445.731924:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941445.731925:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941445.731927:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941445.731938:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941445.731940:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941445.731942:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941445.731946:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60700 x1860112241149440/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941455 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941445.731960:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60700 x1860112241149440/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941455 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941445.731965:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941445.731967:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941445.731968:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941445.731970:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941445.731971:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941445.731972:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941445.731974:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941445.731976:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=24 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941445.731979:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941445.731981:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941445.731982:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941445.731983:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941445.731985:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941445.731987:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941445.731989:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60700 x1860112241149440/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941455 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941445.731994:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941445.731996:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941445.731997:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941445.731999:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941445.732002:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941445.732003:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941445.732004:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941445.732005:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941445.732007:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941445.732008:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941445.732010:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941445.732017:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941445 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941445.732019:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941445.732024:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941445.732025:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941445.732027:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941445.732028:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941445.732029:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941445.732032:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941445.732033:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941445.732035:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941445.732038:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941445.732040:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941445.732043:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941445.732239:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941445.732242:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941445.732246:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60700 x1860112241149440/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941455 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941445.732253:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241149440:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941445.732255:0:12171:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941445.732257:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941445.732260:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1773941445.732266:0:12171:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000040:2.0:1773941445.732267:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60700 x1860112241149440/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941455 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941445.732272:0:12171:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941445.732276:0:12171:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941445.732277:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 10000000:01000000:1.0:1773941445.732278:0:12171:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941445.732280:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941445.732282:0:12171:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941445.732283:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce6000. 00000001:00000001:1.0:1773941445.732286:0:12171:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941445.732287:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941445.732288:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941445.732289:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941445.732290:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941445.732292:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0735c00. 02000000:00000001:2.0:1773941445.732295:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:1.0:1773941445.732296:0:12171:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941445.732299:0:12171:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941445.732299:0:12171:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941445.732301:0:12171:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941445.732304:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:00000001:1.0:1773941445.732304:0:12171:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941445.732307:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60700. 00000100:00000001:1.0:1773941445.732308:0:12171:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941445.732312:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941445.732314:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941445.732314:0:12171:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941445.732318:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941445.732320:0:12171:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941445.732322:0:12171:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941445.732323:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941445.732324:0:12171:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941445.732326:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941445.732327:0:12171:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941445.732329:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941445.732331:0:12171:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941445.732332:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941445.732334:0:12171:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941445.732336:0:12171:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941445.732350:0:12171:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941445.732351:0:12171:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941445.732352:0:12171:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941445.732356:0:12171:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941445.732358:0:12171:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941445.732360:0:12171:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941445.732362:0:12171:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941445.732364:0:12171:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941445.732365:0:12171:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941445.735749:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962300. 00010000:00000010:1.1:1773941445.735762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81484600. 00000020:00000001:0.0:1773941446.495901:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941446.495908:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941446.495913:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941446.751824:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941446.751832:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941446.751837:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941446.751840:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941446.751841:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941446.751843:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941446.751846:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941446.751848:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941446.751850:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941446.751853:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941446.751855:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941446.751866:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277100. 00000020:00000040:3.0:1773941446.751871:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941446.751875:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941446.751877:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941446.751882:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941446.751889:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e3a00. 00000100:00000001:3.0:1773941446.751895:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941446.751898:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941446.751900:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941446.751906:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962700. 00010000:00000001:3.0:1773941446.751911:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941446.751913:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941446.751917:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941446.751919:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1cc2 to hash 00000020:00000001:3.0:1773941446.751921:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941446.751922:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941446.751924:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941446.751928:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941446.751931:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941446.751934:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1cc2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941446.751942:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1cc2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941446.751969:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1cc2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941446.751976:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941446.751978:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941446.751985:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e9c0. 00000100:00000001:3.0:1773941446.751988:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387402176 : -128249322149440 : ffff8b5ba0b5e9c0) 00000100:00000001:3.0:1773941446.751991:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941446.751992:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109629696 : -128249599921920 : ffff8b5b90277100) 00000100:00000001:3.0:1773941446.751997:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941446.752001:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941446.752004:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941446.752006:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941446.752017:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90277100 x1860112241149568/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941446.752024:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941446.752025:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941446.752029:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941446.752031:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941446.752033:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941446.752035:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241149568:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941446.752039:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941446.752043:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941446.752044:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941446.752046:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941446.752050:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941446.752052:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941446.752059:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941446.752061:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abc60. 00000400:00000010:3.0:1773941446.752066:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6af68. 00000100:00000200:3.0:1773941446.752071:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241149568, portal 25 00000100:00000001:3.0:1773941446.752073:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941446.752078:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109629696 : -128249599921920 : ffff8b5b90277100) 00000100:00000040:3.0:1773941446.752082:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90277100 x1860112241149568/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941462 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941446.752088:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941446.752090:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941446.752092:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a7f8. 00000100:00000200:3.0:1773941446.752094:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241149568, offset 0 00000400:00000200:3.0:1773941446.752099:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941446.752117:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941446.752125:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941446.752130:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941446.752134:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941446.752137:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941446.752141:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941446.752144:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[42, 41] 00000400:00000200:3.0:1773941446.752148:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941446.752153:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [148:148:256:4294967295] 192.168.204.153@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:3.0:1773941446.752158:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941446.752168:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941446.752175:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941446.752180:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69700. 00000800:00000200:3.0:1773941446.752187:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941446.752196:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941446.752200:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69700 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941446.752221:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941446.752226:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941446.752229:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294887115 00000100:00000001:3.0:1773941446.752236:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294886093 00000100:00000001:3.0:1773941446.752241:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294887115 00000100:00000001:3.0:1773941446.752245:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294886093 00000100:00000001:3.0:1773941446.752256:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941446.752259:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941446.752262:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941446.752270:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941446.752273:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941446.752277:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941446.752280:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941446.752345:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941446.752349:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69700. 00000400:00000200:2.0:1773941446.752355:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941446.752360:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941446.752365:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a7f8 00000400:00000010:2.0:1773941446.752368:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a7f8. 00000100:00000001:2.0:1773941446.752373:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941446.752379:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277100 x1860112241149568/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941462 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941446.752392:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941446.752397:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90277100 x1860112241149568/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941462 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941446.752404:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941446.752406:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941446.753474:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941446.753485:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941446.753488:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941446.753491:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941446.753497:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941446.753501:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941446.753508:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941446.753513:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941446.753517:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941446.753521:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604680 00000400:00000200:3.0:1773941446.753526:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941446.753531:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xca5 [1] + 224 00000800:00000001:3.0:1773941446.753537:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941446.753544:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941446.753547:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941446.753551:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941446.753556:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941446.753561:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277100 x1860112241149568/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941462 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941446.753572:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90277100 x1860112241149568/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941462 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941446.753638:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941446.753645:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941446.753649:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941446.753678:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941446.753683:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941446.753684:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941446.753686:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abc60. 00000400:00000200:1.0:1773941446.753690:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6af68 00000400:00000010:1.0:1773941446.753691:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6af68. 00000100:00000001:1.0:1773941446.753693:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941446.753697:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277100 x1860112241149568/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941462 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941446.753730:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277100 x1860112241149568/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941462 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941446.753735:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941446.753742:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941446.753744:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941446.753746:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941446.753748:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941446.753750:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941446.753751:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941446.753757:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=25 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941446.753759:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941446.753761:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941446.753764:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941446.753766:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941446.753769:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941446.753771:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941446.753789:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941446.753794:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941446.753797:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941446.753799:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941446.753801:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941446.753802:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941446.753805:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90277100 x1860112241149568/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941462 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941446.753811:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941446.753812:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941446.753813:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941446.753815:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941446.753817:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941446.753818:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941446.753819:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941446.753820:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941446.753821:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941446.753822:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941446.753823:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941446.753825:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941446.753826:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941446.753834:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941446.753836:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 216 00000400:00000200:1.0:1773941446.753840:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941446.753843:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941446.753846:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941446.753848:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941446.753850:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941446.753852:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941446.753854:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941446.753855:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941446.753857:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941446.753859:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941446.753860:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941446.753861:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941446.753863:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941446.753864:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941446.753865:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941446.753867:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941446.753869:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941446.753871:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941446.753873:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941446.753875:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941446.753878:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941446.753880:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941446.753882:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941446.753884:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941446.753887:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63b80. 00000020:00000040:1.0:1773941446.753890:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941446.753893:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941446.753895:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941446.753897:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941446.753899:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941446.753904:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941446.753908:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734c00. 00000100:00000001:1.0:1773941446.753911:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941446.753915:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63b80 x1860112241149696/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941446.753921:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941446.753923:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941446.753926:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000957145d0] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63b80 x1860112241149696/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941446.753953:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941446.753957:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941446.753959:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941446.753963:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941446.753966:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941446.753968:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941446.753971:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941446.753973:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941446.753975:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941446.753980:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90277100 x1860112241149568/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941462 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941446.753989:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941446.753992:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941446.753995:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941446.753997:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941446.754002:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90277100 x1860112241149568/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941462 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941446.754012:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241149568:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941446.754018:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941446.754018:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941446.754023:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941446.754025:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941446.754027:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941446.754030:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941446.754036:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941446.754045:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90277100 x1860112241149568/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941462 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941446.754046:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63b80 x1860112241149696/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941446.754065:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941446.754072:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e9c0. 00000100:00000001:0.0:1773941446.754077:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941446.754078:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941446.754083:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941446.754084:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941446.754087:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941446.754090:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941446.754092:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941446.754094:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 00000020:00000001:1.0:1773941446.754099:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:1.0:1773941446.754104:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 02000000:00000001:0.0:1773941446.754104:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00010000:1.0:1773941446.754109:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1cc2 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941446.754110:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941446.754119:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941446.754127:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00100000:0.0:1773941446.754131:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241149696:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00010000:1.0:1773941446.754134:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1cc2 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941446.754142:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941446.754148:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000001:0.0:1773941446.754150:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00010000:1.0:1773941446.754152:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1cc2 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941446.754155:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941446.754162:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941446.754166:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941446.754170:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 02000000:00000010:0.0:1773941446.754172:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0000. 00010000:00010000:1.0:1773941446.754173:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1cc2 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941446.754180:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941446.754188:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941446.754193:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941446.754193:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941446.754198:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000010:0.0:1773941446.754202:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941446.754210:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00010000:00010000:1.0:1773941446.754211:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1cc2 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:0.0:1773941446.754220:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241149696, portal 25 00010000:00010000:1.0:1773941446.754222:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941446.754227:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:0.0:1773941446.754229:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 10000000:00010000:1.0:1773941446.754231:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1cc2 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941446.754233:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121084800 : -128249588466816 : ffff8b5b90d63b80) 00010000:00000001:1.0:1773941446.754246:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000040:0.0:1773941446.754248:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63b80 x1860112241149696/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941446.754249:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941446.754253:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941446.754256:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000020:00000001:1.0:1773941446.754262:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:1.0:1773941446.754267:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000100:00000001:0.0:1773941446.754272:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941446.754273:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941446.754278:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1cc2 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941446.754280:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941446.754286:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c440. 10000000:00000001:1.0:1773941446.754290:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941446.754295:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1cc2 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:0.0:1773941446.754295:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241149696, offset 0 10000000:01000000:1.0:1773941446.754309:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:0.0:1773941446.754309:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 10000000:00000040:1.0:1773941446.754314:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941446.754317:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941446.754321:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:0.0:1773941446.754322:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941446.754325:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941446.754330:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:0.0:1773941446.754333:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:1.0:1773941446.754334:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941446.754338:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941446.754343:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941446.754345:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:1.0:1773941446.754347:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1cc2 from hash 00010000:00000001:1.0:1773941446.754353:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:0.0:1773941446.754354:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941446.754359:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941446.754364:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941446.754368:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941446.754369:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941446.754372:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941446.754376:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941446.754380:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:0.0:1773941446.754381:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[42, 43] 00010000:00000001:1.0:1773941446.754385:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941446.754389:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:0.0:1773941446.754393:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941446.754394:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941446.754399:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941446.754405:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [149:149:256:4294967295] 192.168.204.143@tcp LPNI seq info [44:44:8:4294967295] 00010000:00010000:1.0:1773941446.754406:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1cc2 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941446.754417:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941446.754419:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941446.754421:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941446.754425:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941446.754427:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941446.754430:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:0.0:1773941446.754430:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941446.754433:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941446.754438:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1cc2 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941446.754452:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000040:1.0:1773941446.754454:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962700 count: 0 00000800:00000200:0.0:1773941446.754460:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941446.754467:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:0.0:1773941446.754468:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00010000:00000001:1.0:1773941446.754471:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941446.754475:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000800:00000200:0.0:1773941446.754480:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000100:00000040:1.0:1773941446.754484:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90277100 x1860112241149568/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941462 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1773941446.754495:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000100:00000001:1.0:1773941446.754498:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941446.754502:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000200:0.0:1773941446.754503:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 02000000:00000010:1.0:1773941446.754507:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 02000000:00000001:1.0:1773941446.754511:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941446.754513:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941446.754515:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941446.754517:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941446.754519:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e3a00. 00000100:00000001:0.0:1773941446.754522:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941446.754530:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:0.0:1773941446.754531:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 02000000:00000001:1.0:1773941446.754534:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941446.754536:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294887117 00000100:00000010:1.0:1773941446.754538:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277100. 00000100:00000001:1.0:1773941446.754545:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941446.754547:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941446.754547:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294887115 10000000:00000001:1.0:1773941446.754552:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941446.754555:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:0.0:1773941446.754557:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294887117 10000000:00000001:1.0:1773941446.754558:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941446.754563:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0cc0. 00000100:00000001:0.0:1773941446.754565:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294887115 00000020:00000010:1.0:1773941446.754569:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000100:00000001:0.0:1773941446.754571:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1773941446.754575:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000100:00000001:0.0:1773941446.754576:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:1.0:1773941446.754580:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000100:00000001:0.0:1773941446.754582:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941446.754585:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000100:00000001:0.0:1773941446.754587:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1773941446.754591:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 00000800:00000200:2.0:1773941446.754597:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1773941446.754597:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 80000000:00000010:1.0:1773941446.754598:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000001:0.0:1773941446.754601:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000010:2.0:1773941446.754603:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00040000:00000010:1.0:1773941446.754606:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 00000100:00000001:0.0:1773941446.754608:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1773941446.754609:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1773941446.754612:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 20000000:00000010:1.0:1773941446.754614:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 00000400:00000200:2.0:1773941446.754616:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941446.754621:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c440 10000000:00000040:1.0:1773941446.754623:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000400:00000010:2.0:1773941446.754624:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c440. 00000100:00000001:0.0:1773941446.754627:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941446.754630:0:2347:0:(events.c:34:request_out_callback()) Process entered 10000000:00000001:1.0:1773941446.754630:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941446.754632:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941446.754637:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63b80 x1860112241149696/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941446.754639:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941446.754647:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000100:00000001:2.0:1773941446.754649:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000010:1.0:1773941446.754652:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000100:00000040:2.0:1773941446.754653:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63b80 x1860112241149696/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941456 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941446.754659:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941446.754665:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941446.754673:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 00000100:00000001:2.0:1773941446.754674:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941446.754678:0:2347:0:(events.c:68:request_out_callback()) Process leaving 20000000:00000010:1.0:1773941446.754679:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941446.754684:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941446.754688:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0cc0. 10000000:01000000:1.0:1773941446.754691:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941446.754694:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941446.754697:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941446.754699:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941446.754701:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941446.754735:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941446.754747:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941446.755919:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941446.755928:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941446.755938:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941446.755942:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941446.755947:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941446.756020:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941446.756028:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941446.756032:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941446.756035:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941446.756039:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604700 00000400:00000200:3.0:1773941446.756043:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941446.756048:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xcb5 [1] + 0 00000800:00000001:3.0:1773941446.756054:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941446.756061:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941446.756064:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941446.756067:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941446.756072:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941446.756077:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63b80 x1860112241149696/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941456 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941446.756089:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63b80 x1860112241149696/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941456 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941446.756107:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941446.756113:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941446.756116:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941446.756158:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941446.756162:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941446.756166:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941446.756168:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941446.756170:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941446.756175:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941446.756176:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941446.756178:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941446.756183:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63b80 x1860112241149696/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941456 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941446.756201:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63b80 x1860112241149696/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941456 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941446.756206:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941446.756207:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941446.756209:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941446.756210:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941446.756211:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941446.756212:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941446.756214:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941446.756217:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=25 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941446.756219:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941446.756221:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941446.756223:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941446.756224:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941446.756226:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941446.756227:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941446.756230:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63b80 x1860112241149696/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941456 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941446.756234:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941446.756236:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941446.756237:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941446.756239:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941446.756243:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941446.756245:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941446.756246:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941446.756247:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941446.756249:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941446.756251:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941446.756252:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941446.756255:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941446 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941446.756257:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941446.756262:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941446.756269:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941446.756271:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941446.756273:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941446.756275:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941446.756279:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941446.756282:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941446.756284:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941446.756288:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941446.756291:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941446.756294:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941446.756519:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941446.756523:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941446.756528:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63b80 x1860112241149696/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941456 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941446.756536:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241149696:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941446.756540:0:12172:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941446.756541:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941446.756545:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941446.756548:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63b80 x1860112241149696/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941456 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941446.756550:0:12172:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941446.756555:0:12172:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941446.756558:0:12172:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941446.756561:0:12172:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941446.756562:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941446.756565:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941446.756565:0:12172:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941446.756568:0:12172:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941446.756570:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0000. 00000001:00000001:0.0:1773941446.756572:0:12172:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941446.756576:0:12172:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941446.756577:0:12172:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941446.756578:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941446.756579:0:12172:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941446.756582:0:12172:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941446.756583:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941446.756584:0:12172:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:2.0:1773941446.756588:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00080000:0.0:1773941446.756590:0:12172:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000020:00000001:2.0:1773941446.756594:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941446.756594:0:12172:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941446.756598:0:12172:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000010:2.0:1773941446.756600:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734c00. 00000100:00000001:0.0:1773941446.756601:0:12172:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941446.756605:0:12172:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 02000000:00000001:2.0:1773941446.756607:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1773941446.756608:0:12172:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941446.756611:0:12172:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 02000000:00000001:2.0:1773941446.756612:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:00000040:0.0:1773941446.756613:0:12172:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000010:2.0:1773941446.756617:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63b80. 00000100:00000001:2.0:1773941446.756622:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941446.756624:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941446.756627:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941446.756627:0:12172:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941446.756630:0:12172:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941446.756632:0:12172:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941446.756633:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:02000000:0.0:1773941446.756635:0:12172:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941446.756637:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941446.756641:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941446.756645:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941446.756645:0:12172:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941446.756649:0:12172:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941446.756651:0:12172:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941446.756653:0:12172:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941446.756654:0:12172:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941446.759745:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962700. 00010000:00000010:1.1:1773941446.759754:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941447.519904:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941447.519910:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941447.519914:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941447.583848:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1773941447.583857:0:37:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1773941447.583866:0:37:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b850a8a80. 00000020:00000040:0.0:1773941447.583871:0:37:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941447.583875:0:37:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941447.583878:0:37:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941447.583883:0:37:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941447.583890:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b99101000. 00000100:00000001:0.0:1773941447.583896:0:37:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941447.583906:0:37:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS req@ffff8b5b850a8a80 x1860112241149824/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941447.583915:0:37:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941447.583918:0:37:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941447.583924:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d2ea9673] to pc [ptlrpcd_00_00+0] req@ffff8b5b850a8a80 x1860112241149824/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941447.583952:0:37:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941447.583955:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (223) 00000100:00000001:0.0:1773941447.583967:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1773941447.583970:0:2353:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1773941447.583972:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941447.583984:0:2353:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941447.583986:0:2353:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941447.583988:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941447.583991:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941447.583991:0:2353:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941447.583994:0:2353:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941447.583996:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941447.583999:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b850a8a80 x1860112241149824/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941447.584006:0:2353:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941447.584021:0:2353:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941447.584024:0:2353:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941447.584026:0:2353:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941447.584028:0:2353:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941447.584031:0:2353:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2353:1860112241149824:192.168.204.143@tcp:400:kworker.0 00000100:00000001:0.0:1773941447.584035:0:2353:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941447.584039:0:2353:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941447.584042:0:2353:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941447.584045:0:2353:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941447.584050:0:2353:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b845c4a00. 02000000:00000001:0.0:1773941447.584052:0:2353:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941447.584059:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941447.584063:0:2353:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941447.584070:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941447.584077:0:2353:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241149824, portal 25 00000100:00000001:0.0:1773941447.584080:0:2353:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941447.584082:0:2353:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615823923186304 : -128249786365312 : ffff8b5b850a8a80) 00000100:00000040:0.0:1773941447.584088:0:2353:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b850a8a80 x1860112241149824/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941447.584096:0:2353:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941447.584100:0:2353:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941447.584103:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c908. 00000100:00000200:0.0:1773941447.584108:0:2353:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241149824, offset 0 00000400:00000200:0.0:1773941447.584112:0:2353:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941447.584119:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941447.584125:0:2353:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941447.584129:0:2353:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941447.584132:0:2353:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941447.584135:0:2353:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941447.584137:0:2353:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941447.584140:0:2353:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[44, 43] 00000400:00000200:0.0:1773941447.584142:0:2353:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:0.0:1773941447.584145:0:2353:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [150:150:256:4294967295] 192.168.204.153@tcp LPNI seq info [45:45:8:4294967295] 00000400:00000200:0.0:1773941447.584153:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:0.0:1773941447.584161:0:2353:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:0.0:1773941447.584165:0:2353:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:0.0:1773941447.584169:0:2353:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b91e1e300. 00000800:00000200:0.0:1773941447.584174:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:0.0:1773941447.584180:0:2353:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:0.0:1773941447.584184:0:2353:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b91e1e300 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1773941447.584197:0:2353:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941447.584200:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941447.584202:0:2353:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294887947 00000100:00000001:0.0:1773941447.584208:0:2353:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294887117 00000100:00000001:0.0:1773941447.584210:0:2353:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294887947 00000100:00000001:0.0:1773941447.584213:0:2353:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294887117 00000100:00000001:0.0:1773941447.584216:0:2353:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941447.584218:0:2353:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941447.584221:0:2353:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941447.584222:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941447.584308:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941447.584315:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b91e1e300. 00000400:00000200:2.0:1773941447.584321:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941447.584334:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941447.584340:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c908 00000400:00000010:2.0:1773941447.584344:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c908. 00000100:00000001:2.0:1773941447.584358:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941447.584364:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b850a8a80 x1860112241149824/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941447.584374:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941447.584378:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b850a8a80 x1860112241149824/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941463 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941447.584386:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941447.584388:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941447.585629:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941447.585641:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941447.585643:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941447.585645:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941447.585649:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941447.585653:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941447.585691:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941447.585694:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941447.585727:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941447.585732:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604780 00000400:00000200:3.0:1773941447.585736:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941447.585742:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xcbd [1] + 224 00000800:00000001:3.0:1773941447.585749:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941447.585775:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941447.585779:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941447.585783:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941447.585787:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941447.585793:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b850a8a80 x1860112241149824/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941463 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941447.585804:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b850a8a80 x1860112241149824/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941463 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941447.585824:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941447.585832:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941447.585835:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941447.585870:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941447.585874:0:2353:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941447.585880:0:2353:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941447.585882:0:2353:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941447.585885:0:2353:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941447.585890:0:2353:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941447.585892:0:2353:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941447.585895:0:2353:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941447.585901:0:2353:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b850a8a80 x1860112241149824/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941463 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941447.585913:0:2353:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b850a8a80 x1860112241149824/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941463 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941447.585921:0:2353:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941447.585922:0:2353:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941447.585932:0:2353:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941447.585935:0:2353:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941447.585937:0:2353:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941447.585938:0:2353:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941447.585940:0:2353:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941447.585944:0:2353:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=26 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941447.585947:0:2353:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941447.585949:0:2353:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941447.585950:0:2353:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941447.585952:0:2353:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941447.585955:0:2353:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941447.585958:0:2353:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941447.585962:0:2353:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941447.585965:0:2353:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941447.585967:0:2353:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941447.585968:0:2353:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941447.585969:0:2353:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941447.585973:0:2353:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b850a8a80 x1860112241149824/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941463 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941447.585981:0:2353:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941447.585982:0:2353:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941447.585984:0:2353:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941447.585987:0:2353:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941447.585990:0:2353:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941447.585992:0:2353:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941447.585993:0:2353:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941447.585995:0:2353:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941447.585996:0:2353:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941447.585997:0:2353:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941447.585999:0:2353:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941447.586001:0:2353:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941447.586003:0:2353:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941447.586007:0:2353:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941447.586010:0:2353:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 217 00000400:00000200:0.0:1773941447.586022:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941447.586027:0:2353:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941447.586031:0:2353:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941447.586035:0:2353:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941447.586037:0:2353:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941447.586039:0:2353:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:0.0:1773941447.586043:0:2353:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941447.586045:0:2353:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941447.586048:0:2353:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941447.586051:0:2353:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941447.586052:0:2353:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941447.586054:0:2353:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:0.0:1773941447.586057:0:2353:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941447.586058:0:2353:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941447.586059:0:2353:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941447.586060:0:2353:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:0.0:1773941447.586062:0:2353:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:0.0:1773941447.586064:0:2353:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:0.0:1773941447.586066:0:2353:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941447.586068:0:2353:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941447.586070:0:2353:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941447.586072:0:2353:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941447.586074:0:2353:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941447.586075:0:2353:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941447.586078:0:2353:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b850a8380. 00000020:00000040:0.0:1773941447.586080:0:2353:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:0.0:1773941447.586082:0:2353:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941447.586083:0:2353:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941447.586085:0:2353:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941447.586087:0:2353:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941447.586089:0:2353:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941447.586092:0:2353:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b850c3000. 00000100:00000001:0.0:1773941447.586099:0:2353:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941447.586104:0:2353:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b850a8380 x1860112241149952/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941447.586110:0:2353:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941447.586112:0:2353:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941447.586115:0:2353:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ed9c4e6] to pc [ptlrpcd_rcv+-1] req@ffff8b5b850a8380 x1860112241149952/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941447.586133:0:2353:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941447.586135:0:2353:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941447.586136:0:2353:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:0.0:1773941447.586140:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b850a8a80 x1860112241149824/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941463 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941447.586145:0:2353:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941447.586147:0:2353:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941447.586149:0:2353:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941447.586152:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b850a8a80 x1860112241149824/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941463 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941447.586158:0:2353:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2353:1860112241149824:192.168.204.143@tcp:400:kworker.0 00000100:00000001:0.0:1773941447.586162:0:2353:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941447.586163:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941447.586163:0:2353:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941447.586167:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1773941447.586169:0:2353:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b850a8a80 x1860112241149824/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941463 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941447.586170:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1773941447.586178:0:2353:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000040:2.0:1773941447.586179:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b850a8380 x1860112241149952/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1773941447.586181:0:2353:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941447.586184:0:2353:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b845c4a00. 02000000:00000001:0.0:1773941447.586189:0:2353:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1773941447.586192:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1773941447.586193:0:2353:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941447.586195:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941447.586196:0:2353:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 02000000:00000001:2.0:1773941447.586201:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1773941447.586201:0:2353:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:2.0:1773941447.586204:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941447.586206:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941447.586209:0:2353:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b99101000. 00000100:00100000:2.0:1773941447.586211:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241149952:192.168.204.143@tcp:250:ptlrpcd_00_01.0 02000000:00000001:0.0:1773941447.586215:0:2353:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1773941447.586216:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941447.586219:0:2353:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1773941447.586221:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:0.0:1773941447.586222:0:2353:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b850a8a80. 02000000:00000001:2.0:1773941447.586223:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941447.586227:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1773941447.586227:0:2353:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941447.586230:0:2353:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1773941447.586233:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce5c00. 00000100:00000001:0.0:1773941447.586234:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941447.586239:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941447.586242:0:2353:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941447.586245:0:2353:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941447.586246:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941447.586250:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:2.0:1773941447.586251:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000100:00000001:0.0:1773941447.586253:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1773941447.586256:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000200:2.0:1773941447.586261:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241149952, portal 25 00000100:00000001:2.0:1773941447.586263:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941447.586270:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615823923184512 : -128249786367104 : ffff8b5b850a8380) 00000100:00000040:2.0:1773941447.586274:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b850a8380 x1860112241149952/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941457 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941447.586279:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941447.586281:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941447.586283:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f343b8. 00000100:00000200:2.0:1773941447.586284:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241149952, offset 0 00000400:00000200:2.0:1773941447.586287:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941447.586290:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941447.586293:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941447.586295:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941447.586298:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941447.586299:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941447.586301:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941447.586303:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[44, 45] 00000400:00000200:2.0:1773941447.586306:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941447.586308:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [151:151:256:4294967295] 192.168.204.143@tcp LPNI seq info [46:46:8:4294967295] 00000400:00000200:2.0:1773941447.586310:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941447.586315:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941447.586318:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941447.586321:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0200. 00000800:00000200:2.0:1773941447.586324:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941447.586329:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941447.586331:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0200 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941447.586342:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941447.586344:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941447.586346:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294887949 00000100:00000001:2.0:1773941447.586349:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294887947 00000100:00000001:2.0:1773941447.586352:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294887949 00000100:00000001:2.0:1773941447.586354:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294887947 00000100:00000001:2.0:1773941447.586360:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941447.586362:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941447.586363:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941447.586365:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941447.586368:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941447.586369:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941447.586370:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941447.586371:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941447.586372:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941447.586373:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941447.586439:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941447.586442:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0200. 00000400:00000200:3.0:1773941447.586447:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941447.586451:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941447.586453:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f343b8 00000400:00000010:3.0:1773941447.586455:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f343b8. 00000100:00000001:3.0:1773941447.586458:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941447.586461:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b850a8380 x1860112241149952/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941457 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941447.586468:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941447.586470:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b850a8380 x1860112241149952/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941457 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941447.586475:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941447.586476:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941447.587720:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941447.587726:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941447.587734:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941447.587781:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941447.587786:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941447.587790:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941447.587794:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941447.587796:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941447.587799:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941447.587801:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604800 00000400:00000200:3.0:1773941447.587804:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941447.587807:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xcc5 [1] + 0 00000800:00000001:3.0:1773941447.587810:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941447.587815:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941447.587816:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941447.587819:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941447.587822:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941447.587825:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b850a8380 x1860112241149952/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941457 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941447.587833:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b850a8380 x1860112241149952/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941457 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941447.587845:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941447.587849:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941447.587852:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941447.587874:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941447.587877:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941447.587881:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941447.587882:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941447.587885:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941447.587889:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941447.587891:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941447.587894:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941447.587900:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b850a8380 x1860112241149952/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941457 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941447.587909:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b850a8380 x1860112241149952/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941457 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941447.587922:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941447.587924:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941447.587926:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941447.587928:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941447.587929:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941447.587931:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941447.587933:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941447.587936:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=26 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941447.587940:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941447.587943:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941447.587944:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941447.587947:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941447.587949:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941447.587952:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941447.587966:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b850a8380 x1860112241149952/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941457 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941447.587973:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941447.587975:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941447.587977:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941447.587979:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941447.587985:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941447.587987:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941447.587988:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941447.587990:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941447.587992:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941447.587995:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941447.587997:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941447.588001:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941447 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941447.588004:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941447.588010:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941447.588012:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941447.588014:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941447.588016:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941447.588018:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941447.588021:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941447.588024:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941447.588026:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941447.588029:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941447.588032:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941447.588034:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941447.588300:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941447.588304:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941447.588309:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b850a8380 x1860112241149952/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941457 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941447.588317:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241149952:192.168.204.143@tcp:250:ptlrpcd_00_01.0 00000100:00000001:0.0:1773941447.588322:0:12173:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941447.588329:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941447.588332:0:12173:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:2.0:1773941447.588333:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:0.0:1773941447.588338:0:12173:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:2.0:1773941447.588339:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b850a8380 x1860112241149952/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941457 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941447.588341:0:12173:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941447.588344:0:12173:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941447.588347:0:12173:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941447.588350:0:12173:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941447.588353:0:12173:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941447.588355:0:12173:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941447.588356:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941447.588356:0:12173:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941447.588360:0:12173:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941447.588361:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 10000000:00000001:0.0:1773941447.588363:0:12173:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941447.588364:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce5c00. 00000100:00000001:0.0:1773941447.588366:0:12173:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:2.0:1773941447.588370:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:0.0:1773941447.588371:0:12173:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000020:00000001:2.0:1773941447.588373:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941447.588376:0:12173:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000040:2.0:1773941447.588377:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941447.588379:0:12173:0:(recover.c:147:ptlrpc_resend()) Process entered 00000020:00000001:2.0:1773941447.588381:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941447.588382:0:12173:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941447.588384:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b850c3000. 00000100:00080000:0.0:1773941447.588385:0:12173:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 02000000:00000001:2.0:1773941447.588389:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1773941447.588390:0:12173:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941447.588393:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:01000000:0.0:1773941447.588394:0:12173:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000010:2.0:1773941447.588396:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b850a8380. 10000000:00000040:0.0:1773941447.588397:0:12173:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:2.0:1773941447.588402:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941447.588404:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941447.588406:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941447.588409:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941447.588410:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941447.588412:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941447.588413:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941447.588414:0:12173:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941447.588417:0:12173:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941447.588418:0:12173:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941447.588422:0:12173:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941447.588426:0:12173:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941447.588433:0:12173:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941447.588434:0:12173:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941447.588436:0:12173:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941447.588438:0:12173:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941448.159812:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941448.159821:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941448.159825:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941448.159828:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941448.159830:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941448.159831:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941448.159834:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941448.159836:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941448.159838:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941448.159841:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941448.159844:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941448.159854:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60a80. 00000020:00000040:1.0:1773941448.159859:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941448.159865:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941448.159868:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941448.159873:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941448.159880:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941448.159887:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941448.159891:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941448.159894:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941448.159901:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ecd00. 00010000:00000001:1.0:1773941448.159906:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941448.159910:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138e600. 00000020:00000001:1.0:1773941448.159914:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941448.159917:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138e600 with handle 0xa33a5be14eae1cc9 to hash 00000020:00000001:1.0:1773941448.159919:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941448.159921:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941448.159925:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941448.159929:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941448.159932:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941448.159937:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cc9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941448.159968:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cc9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941448.159980:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cc9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941448.159986:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941448.159988:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941448.160005:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941448.160008:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941448.160010:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941448.160012:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121072256 : -128249588479360 : ffff8b5b90d60a80) 00000100:00000001:1.0:1773941448.160017:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941448.160022:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941448.160024:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941448.160026:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941448.160036:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60a80 x1860112241150080/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941448.160045:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941448.160047:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941448.160051:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941448.160054:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941448.160057:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941448.160060:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241150080:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941448.160065:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941448.160069:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941448.160072:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941448.160074:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941448.160080:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734c00. 02000000:00000001:1.0:1773941448.160082:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941448.160089:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941448.160099:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941448.160114:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941448.160121:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241150080, portal 25 00000100:00000001:1.0:1773941448.160127:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941448.160128:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121072256 : -128249588479360 : ffff8b5b90d60a80) 00000100:00000040:1.0:1773941448.160134:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60a80 x1860112241150080/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941464 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941448.160142:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941448.160144:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941448.160147:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1110. 00000100:00000200:1.0:1773941448.160150:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241150080, offset 0 00000400:00000200:1.0:1773941448.160156:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941448.160164:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941448.160171:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941448.160177:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941448.160181:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941448.160186:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941448.160189:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941448.160192:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[46, 45] 00000400:00000200:1.0:1773941448.160196:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941448.160200:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [152:152:256:4294967295] 192.168.204.153@tcp LPNI seq info [47:47:8:4294967295] 00000400:00000200:1.0:1773941448.160205:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941448.160215:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941448.160221:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941448.160227:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b800. 00000800:00000200:1.0:1773941448.160233:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941448.160241:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941448.160245:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b800 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941448.160265:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941448.160269:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941448.160272:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294888523 00000100:00000001:1.0:1773941448.160284:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294887949 00000100:00000001:1.0:1773941448.160291:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294888523 00000100:00000001:1.0:1773941448.160295:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294887949 00000100:00000001:1.0:1773941448.160299:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941448.160302:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941448.160305:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941448.160307:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941448.160310:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941448.160313:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941448.160316:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941448.160381:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941448.160385:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b800. 00000400:00000200:3.0:1773941448.160391:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941448.160395:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941448.160399:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1110 00000400:00000010:3.0:1773941448.160401:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1110. 00000100:00000001:3.0:1773941448.160405:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941448.160408:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60a80 x1860112241150080/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941464 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941448.160417:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941448.160425:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d60a80 x1860112241150080/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941464 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941448.160431:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941448.160432:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941448.161687:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941448.161699:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941448.161727:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941448.161731:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941448.161958:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941448.161964:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941448.161972:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941448.161976:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941448.161982:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941448.161987:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604880 00000400:00000200:3.0:1773941448.162001:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941448.162007:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xcd5 [1] + 224 00000800:00000001:3.0:1773941448.162014:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941448.162024:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941448.162027:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941448.162030:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941448.162036:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941448.162042:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60a80 x1860112241150080/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941464 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941448.162053:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d60a80 x1860112241150080/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941464 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941448.162071:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941448.162078:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941448.162082:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941448.162119:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941448.162127:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941448.162129:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941448.162133:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941448.162138:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941448.162140:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941448.162143:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941448.162149:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60a80 x1860112241150080/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941464 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941448.162162:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60a80 x1860112241150080/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941464 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941448.162170:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941448.162172:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941448.162175:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941448.162178:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941448.162180:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941448.162181:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941448.162184:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941448.162188:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=27 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941448.162191:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941448.162193:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941448.162196:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941448.162199:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941448.162208:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941448.162210:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941448.162214:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941448.162217:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941448.162220:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941448.162222:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941448.162223:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941448.162225:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941448.162227:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d60a80 x1860112241150080/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941464 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941448.162233:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941448.162233:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941448.162235:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941448.162237:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941448.162239:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941448.162240:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941448.162241:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941448.162242:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941448.162243:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941448.162244:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941448.162245:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941448.162247:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941448.162248:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941448.162251:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941448.162253:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 218 00000400:00000200:1.0:1773941448.162258:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941448.162262:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941448.162266:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941448.162269:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941448.162272:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941448.162274:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941448.162278:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941448.162285:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941448.162288:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941448.162291:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941448.162293:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941448.162296:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941448.162300:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941448.162303:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941448.162305:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941448.162307:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941448.162310:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941448.162314:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941448.162317:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941448.162319:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941448.162323:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941448.162325:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941448.162327:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941448.162329:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941448.162332:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61c00. 00000020:00000040:1.0:1773941448.162335:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941448.162337:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941448.162338:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941448.162340:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941448.162342:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941448.162344:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941448.162348:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0735c00. 00000100:00000001:1.0:1773941448.162352:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941448.162356:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d61c00 x1860112241150208/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941448.162363:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941448.162366:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941448.162369:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbfa947b] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d61c00 x1860112241150208/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941448.162395:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941448.162397:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941448.162399:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941448.162401:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941448.162403:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941448.162404:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941448.162405:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941448.162407:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941448.162409:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941448.162412:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60a80 x1860112241150080/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941464 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941448.162417:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941448.162419:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941448.162421:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941448.162422:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941448.162425:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60a80 x1860112241150080/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941464 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941448.162431:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241150080:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941448.162435:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941448.162437:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941448.162438:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941448.162440:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941448.162442:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60a80 x1860112241150080/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941464 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941448.162447:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941448.162448:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941448.162453:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941448.162456:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000010:1.0:1773941448.162462:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000040:2.0:1773941448.162466:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d61c00 x1860112241150208/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941448.162466:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941448.162470:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941448.162475:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941448.162484:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941448.162486:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941448.162487:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1773941448.162488:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138e600 refcount=4 00000100:00000001:2.0:1773941448.162493:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941448.162493:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941448.162499:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 02000000:00000001:2.0:1773941448.162501:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00010000:1.0:1773941448.162504:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cc9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941448.162505:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941448.162512:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941448.162519:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00100000:2.0:1773941448.162522:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241150208:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00010000:1.0:1773941448.162524:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cc9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941448.162532:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941448.162534:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941448.162537:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cc9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941448.162542:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941448.162545:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 02000000:00000001:2.0:1773941448.162546:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00010000:1.0:1773941448.162547:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cc9 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941448.162551:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941448.162554:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 02000000:00000001:2.0:1773941448.162556:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941448.162557:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941448.162561:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 02000000:00000010:2.0:1773941448.162564:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce5c00. 00010000:00010000:1.0:1773941448.162564:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cc9 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941448.162570:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941448.162573:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138e600 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941448.162576:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941448.162579:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00010000:1.0:1773941448.162580:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cc9 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941448.162587:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941448.162594:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00000001:1.0:1773941448.162596:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941448.162599:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000200:2.0:1773941448.162600:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241150208, portal 25 00000020:00000001:1.0:1773941448.162601:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941448.162603:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138e600 refcount=5 00000100:00000001:2.0:1773941448.162606:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000020:00000001:1.0:1773941448.162607:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00000100:00000001:2.0:1773941448.162612:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121076736 : -128249588474880 : ffff8b5b90d61c00) 00010000:00000001:1.0:1773941448.162612:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941448.162619:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941448.162622:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cc9 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941448.162624:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d61c00 x1860112241150208/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941458 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941448.162631:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941448.162634:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cc9 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941448.162646:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 10000000:01000000:1.0:1773941448.162648:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941448.162652:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000040:2.0:1773941448.162653:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 10000000:00000001:1.0:1773941448.162654:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941448.162659:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000010:2.0:1773941448.162660:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34088. 00010000:00000001:1.0:1773941448.162661:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941448.162664:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000200:2.0:1773941448.162667:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241150208, offset 0 00010000:00000001:1.0:1773941448.162667:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941448.162670:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941448.162672:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941448.162675:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138e600 with handle 0xa33a5be14eae1cc9 from hash 00010000:00000001:1.0:1773941448.162678:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:2.0:1773941448.162679:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941448.162680:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941448.162682:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941448.162685:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941448.162686:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941448.162687:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:2.0:1773941448.162692:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941448.162693:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941448.162695:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941448.162698:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941448.162725:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941448.162728:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cc9 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941448.162732:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941448.162733:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941448.162735:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941448.162736:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941448.162738:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941448.162739:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941448.162740:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941448.162741:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cc9 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941448.162742:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000040:1.0:1773941448.162760:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ecd00 count: 0 00000400:00000200:2.0:1773941448.162764:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941448.162770:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941448.162775:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941448.162775:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941448.162779:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941448.162782:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1773941448.162783:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941448.162790:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60a80 x1860112241150080/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941464 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941448.162794:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[46, 47] 00000100:00000001:1.0:1773941448.162807:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000400:00000200:2.0:1773941448.162808:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 02000000:00000001:1.0:1773941448.162810:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941448.162813:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734c00. 00000400:00000200:2.0:1773941448.162817:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [153:153:256:4294967295] 192.168.204.143@tcp LPNI seq info [48:48:8:4294967295] 02000000:00000001:1.0:1773941448.162818:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941448.162821:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941448.162824:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941448.162828:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000200:2.0:1773941448.162829:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 02000000:00000010:1.0:1773941448.162830:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941448.162836:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941448.162837:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941448.162840:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60a80. 00000100:00000001:1.0:1773941448.162844:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000400:00000200:2.0:1773941448.162847:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000100:00000001:1.0:1773941448.162848:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941448.162852:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941448.162857:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941448.162860:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000800:00000200:2.0:1773941448.162862:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941448.162868:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0800. 10000000:00000010:1.0:1773941448.162871:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0cc0. 00000800:00000200:2.0:1773941448.162878:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000020:00000010:1.0:1773941448.162878:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941448.162886:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941448.162892:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000800:00000200:2.0:1773941448.162897:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000020:00000010:1.0:1773941448.162898:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000800:00000200:2.0:1773941448.162903:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0800 type 1, nob 616 niov 1 nkiov 1 00000020:00000010:1.0:1773941448.162906:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941448.162914:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000001:2.0:1773941448.162920:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:1.0:1773941448.162920:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 00000400:00000200:2.0:1773941448.162927:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941448.162930:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294888526 00000100:00000001:2.0:1773941448.162937:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294888523 20000000:00000010:1.0:1773941448.162938:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 00000100:00000001:2.0:1773941448.162943:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294888526 10000000:00000040:1.0:1773941448.162949:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000100:00000001:2.0:1773941448.162950:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294888523 10000000:00000001:1.0:1773941448.162953:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941448.162957:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1773941448.162959:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000100:00000001:2.0:1773941448.162963:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:1.0:1773941448.162964:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000100:00000001:2.0:1773941448.162969:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941448.162969:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000100:00000001:2.0:1773941448.162976:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1773941448.162976:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941448.162980:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00000100:00000001:2.0:1773941448.162985:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00040000:00000010:1.0:1773941448.162986:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 00000100:00000001:2.0:1773941448.162989:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 20000000:00000010:1.0:1773941448.162991:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000100:00000001:2.0:1773941448.162995:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:1.0:1773941448.162996:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 00000100:00000001:2.0:1773941448.162999:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000010:1.0:1773941448.163002:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0cc0. 00000100:00000001:2.0:1773941448.163003:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941448.163009:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941448.163009:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000800:00000200:3.0:1773941448.163010:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 10000000:00000001:1.0:1773941448.163016:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000010:3.0:1773941448.163019:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0800. 10000000:00020000:1.0:1773941448.163022:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941448.163026:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 00000400:00000200:3.0:1773941448.163027:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000001:1.0:1773941448.163029:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941448.163032:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 00000400:00000200:3.0:1773941448.163038:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 10000000:00000001:1.0:1773941448.163038:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000400:00000200:3.0:1773941448.163050:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34088 00000400:00000010:3.0:1773941448.163053:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34088. 00000100:00000001:3.0:1773941448.163057:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941448.163063:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61c00 x1860112241150208/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941458 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941448.163072:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941448.163075:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d61c00 x1860112241150208/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941458 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941448.163083:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941448.163085:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941448.164337:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941448.164345:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941448.164347:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941448.164348:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941448.164352:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941448.164356:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941448.164361:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941448.164363:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941448.164366:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941448.164368:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604900 00000400:00000200:3.0:1773941448.164372:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941448.164375:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xcdd [1] + 0 00000800:00000001:3.0:1773941448.164379:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941448.164383:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941448.164434:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941448.164438:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941448.164441:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941448.164446:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61c00 x1860112241150208/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941458 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941448.164454:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d61c00 x1860112241150208/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941458 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941448.164466:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941448.164477:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941448.164480:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941448.164505:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941448.164509:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941448.164513:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941448.164515:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941448.164518:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941448.164523:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941448.164526:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941448.164528:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941448.164534:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61c00 x1860112241150208/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941458 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941448.164545:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61c00 x1860112241150208/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941458 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941448.164553:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941448.164554:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941448.164557:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941448.164566:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941448.164568:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941448.164570:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941448.164572:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941448.164577:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=27 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941448.164581:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941448.164583:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941448.164585:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941448.164588:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941448.164590:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941448.164592:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941448.164598:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d61c00 x1860112241150208/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941458 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941448.164606:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941448.164608:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941448.164610:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941448.164612:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941448.164617:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941448.164620:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941448.164621:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941448.164623:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941448.164626:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941448.164628:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941448.164630:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941448.164634:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941448 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941448.164637:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941448.164643:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941448.164646:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941448.164648:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941448.164650:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941448.164652:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941448.164655:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941448.164658:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941448.164661:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941448.164670:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941448.164673:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941448.164676:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941448.164935:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941448.164940:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941448.164945:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d61c00 x1860112241150208/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941458 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941448.164956:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241150208:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941448.164957:0:12174:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941448.164965:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941448.164968:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941448.164974:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d61c00 x1860112241150208/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941458 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941448.164981:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00080000:1.0:1773941448.164982:0:12174:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 02000000:00000001:2.0:1773941448.164985:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:1.0:1773941448.164988:0:12174:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000010:2.0:1773941448.164990:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce5c00. 00000100:00000001:1.0:1773941448.164993:0:12174:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941448.164996:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:01000000:1.0:1773941448.164997:0:12174:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000001:2.0:1773941448.165000:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:1.0:1773941448.165002:0:12174:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000040:2.0:1773941448.165004:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:1.0:1773941448.165008:0:12174:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:2.0:1773941448.165009:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941448.165011:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0735c00. 02000000:00000001:2.0:1773941448.165014:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941448.165018:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941448.165020:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61c00. 00000001:00000001:1.0:1773941448.165021:0:12174:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941448.165024:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:1.0:1773941448.165026:0:12174:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941448.165028:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941448.165030:0:12174:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941448.165034:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941448.165035:0:12174:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941448.165041:0:12174:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941448.165044:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941448.165044:0:12174:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941448.165049:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941448.165052:0:12174:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941448.165054:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941448.165058:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941448.165059:0:12174:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941448.165063:0:12174:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941448.165065:0:12174:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941448.165068:0:12174:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941448.165071:0:12174:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941448.165073:0:12174:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941448.165075:0:12174:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941448.165107:0:12174:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941448.165108:0:12174:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941448.165111:0:12174:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941448.165116:0:12174:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941448.165119:0:12174:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941448.165122:0:12174:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941448.165124:0:12174:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941448.165127:0:12174:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941448.165129:0:12174:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941448.168762:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ecd00. 00010000:00000010:1.1:1773941448.168775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138e600. 00000020:00000001:0.0:1773941448.543891:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941448.543896:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941448.543907:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941449.567864:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941449.567868:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941449.567872:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941449.951822:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941449.951829:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941449.951832:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941449.951834:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941449.951835:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941449.951836:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941449.951838:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941449.951840:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941449.951842:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941449.951844:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941449.951845:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941449.951854:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277480. 00000020:00000040:3.0:1773941449.951858:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941449.951862:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941449.951865:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941449.951870:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941449.951876:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2a00. 00000100:00000001:3.0:1773941449.951882:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941449.951885:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941449.951888:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941449.951894:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962a00. 00010000:00000001:3.0:1773941449.951899:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941449.951901:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485000. 00000020:00000001:3.0:1773941449.951904:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941449.951907:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485000 with handle 0xa33a5be14eae1cd0 to hash 00000020:00000001:3.0:1773941449.951908:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941449.951909:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860125696 : -128249849425920 : ffff8b5b81485000) 00010000:00000001:3.0:1773941449.951912:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860125696 : -128249849425920 : ffff8b5b81485000) 00010000:00000001:3.0:1773941449.951915:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941449.951917:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941449.951920:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1cd0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941449.951941:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1cd0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941449.951951:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1cd0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941449.951956:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941449.951958:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941449.951964:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e9c0. 00000100:00000001:3.0:1773941449.951966:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387402176 : -128249322149440 : ffff8b5ba0b5e9c0) 00000100:00000001:3.0:1773941449.951969:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941449.951970:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109630592 : -128249599921024 : ffff8b5b90277480) 00000100:00000001:3.0:1773941449.951975:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941449.951980:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941449.951982:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941449.951984:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941449.951995:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90277480 x1860112241150336/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941449.952004:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941449.952006:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941449.952011:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941449.952013:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941449.952016:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941449.952020:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241150336:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941449.952025:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941449.952030:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941449.952033:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941449.952035:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941449.952040:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90263000. 02000000:00000001:3.0:1773941449.952043:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941449.952050:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941449.952058:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab210. 00000400:00000010:3.0:1773941449.952063:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a550. 00000100:00000200:3.0:1773941449.952069:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241150336, portal 25 00000100:00000001:3.0:1773941449.952070:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941449.952071:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109630592 : -128249599921024 : ffff8b5b90277480) 00000100:00000040:3.0:1773941449.952076:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90277480 x1860112241150336/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941465 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941449.952082:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941449.952084:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941449.952086:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a660. 00000100:00000200:3.0:1773941449.952088:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241150336, offset 0 00000400:00000200:3.0:1773941449.952093:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941449.952100:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941449.952106:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941449.952110:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941449.952114:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941449.952117:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941449.952119:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941449.952122:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[48, 47] 00000400:00000200:3.0:1773941449.952124:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941449.952127:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [154:154:256:4294967295] 192.168.204.153@tcp LPNI seq info [49:49:8:4294967295] 00000400:00000200:3.0:1773941449.952130:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941449.952138:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941449.952142:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941449.952146:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69f00. 00000800:00000200:3.0:1773941449.952151:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941449.952157:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941449.952160:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69f00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941449.952179:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941449.952182:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941449.952184:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294890315 00000100:00000001:3.0:1773941449.952194:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294888526 00000100:00000001:3.0:1773941449.952198:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294890315 00000100:00000001:3.0:1773941449.952200:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294888526 00000100:00000001:3.0:1773941449.952203:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941449.952205:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941449.952207:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941449.952209:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941449.952211:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941449.952214:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941449.952217:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941449.952301:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941449.952306:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69f00. 00000400:00000200:2.0:1773941449.952312:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941449.952324:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941449.952329:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a660 00000400:00000010:2.0:1773941449.952332:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a660. 00000100:00000001:2.0:1773941449.952337:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941449.952343:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277480 x1860112241150336/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941465 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941449.952355:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941449.952359:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90277480 x1860112241150336/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941465 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941449.952367:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941449.952369:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941449.953135:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941449.953143:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941449.953146:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941449.953147:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941449.953151:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941449.953155:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941449.953190:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941449.953194:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941449.953197:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941449.953200:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604980 00000400:00000200:3.0:1773941449.953203:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941449.953206:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xced [1] + 224 00000800:00000001:3.0:1773941449.953211:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941449.953216:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941449.953219:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941449.953222:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941449.953228:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941449.953232:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277480 x1860112241150336/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941465 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941449.953243:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90277480 x1860112241150336/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941465 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941449.953277:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941449.953286:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941449.953288:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941449.953327:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941449.953335:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941449.953338:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941449.953340:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab210. 00000400:00000200:1.0:1773941449.953346:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a550 00000400:00000010:1.0:1773941449.953348:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a550. 00000100:00000001:1.0:1773941449.953351:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941449.953357:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277480 x1860112241150336/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941465 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941449.953368:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277480 x1860112241150336/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941465 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941449.953376:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941449.953384:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941449.953387:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941449.953390:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941449.953392:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941449.953394:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941449.953397:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941449.953402:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=28 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941449.953406:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941449.953410:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941449.953414:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941449.953417:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941449.953421:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941449.953424:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941449.953428:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941449.953431:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941449.953435:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941449.953437:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941449.953439:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941449.953441:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941449.953445:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90277480 x1860112241150336/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941465 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941449.953454:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941449.953456:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941449.953458:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941449.953460:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941449.953463:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941449.953465:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941449.953467:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941449.953469:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941449.953470:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941449.953471:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941449.953473:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941449.953475:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941449.953477:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941449.953492:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941449.953495:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 218 00000400:00000200:1.0:1773941449.953501:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941449.953505:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941449.953509:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941449.953512:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941449.953515:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941449.953518:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941449.953521:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941449.953523:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941449.953526:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941449.953528:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941449.953531:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941449.953533:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941449.953537:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941449.953539:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941449.953541:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941449.953543:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941449.953547:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941449.953550:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941449.953553:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941449.953556:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941449.953559:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941449.953562:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941449.953564:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941449.953567:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941449.953571:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60a80. 00000020:00000040:1.0:1773941449.953575:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941449.953578:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941449.953580:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941449.953582:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941449.953584:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941449.953591:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941449.953596:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734c00. 00000100:00000001:1.0:1773941449.953600:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941449.953607:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60a80 x1860112241150464/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941449.953616:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941449.953619:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941449.953624:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000079bd9b68] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60a80 x1860112241150464/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941449.953645:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941449.953649:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941449.953651:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941449.953654:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941449.953657:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941449.953659:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941449.953662:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941449.953664:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941449.953667:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941449.953667:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941449.953671:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941449.953675:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941449.953677:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90277480 x1860112241150336/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941465 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941449.953683:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60a80 x1860112241150464/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941449.953735:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941449.953737:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941449.953740:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941449.953742:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941449.953744:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941449.953747:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241150464:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941449.953750:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941449.953750:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:2.0:1773941449.953752:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941449.953755:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941449.953758:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941449.953758:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1773941449.953764:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce5400. 00000100:00000001:1.0:1773941449.953764:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1773941449.953769:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941449.953769:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941449.953774:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941449.953778:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000100:00000040:1.0:1773941449.953782:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90277480 x1860112241150336/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941465 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941449.953783:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000200:2.0:1773941449.953788:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241150464, portal 25 00000100:00000001:2.0:1773941449.953792:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00100000:1.0:1773941449.953799:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241150336:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941449.953801:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121072256 : -128249588479360 : ffff8b5b90d60a80) 00000100:00000001:1.0:1773941449.953810:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941449.953811:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60a80 x1860112241150464/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941459 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941449.953816:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941449.953822:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941449.953823:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941449.953826:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941449.953828:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34330. 00000100:00000200:2.0:1773941449.953830:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241150464, offset 0 00000100:00000001:1.0:1773941449.953835:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1773941449.953837:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941449.953843:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000040:1.0:1773941449.953844:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90277480 x1860112241150336/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941465 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941449.953848:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941449.953854:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941449.953859:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941449.953864:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1773941449.953865:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941449.953870:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000100:00000010:1.0:1773941449.953871:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e9c0. 00000400:00000200:2.0:1773941449.953874:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[48, 49] 00000100:00000001:1.0:1773941449.953878:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000400:00000200:2.0:1773941449.953879:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000100:00000001:1.0:1773941449.953883:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941449.953884:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [155:155:256:4294967295] 192.168.204.143@tcp LPNI seq info [50:50:8:4294967295] 00010000:00000001:1.0:1773941449.953890:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000200:2.0:1773941449.953891:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941449.953896:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941449.953900:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941449.953901:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000020:00000040:1.0:1773941449.953906:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485000 refcount=4 00000800:00000200:2.0:1773941449.953907:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000020:00000001:1.0:1773941449.953912:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860125696 : -128249849425920 : ffff8b5b81485000) 00000800:00000010:2.0:1773941449.953913:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0e00. 00010000:00000001:1.0:1773941449.953919:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860125696 : -128249849425920 : ffff8b5b81485000) 00000800:00000200:2.0:1773941449.953922:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00010000:1.0:1773941449.953927:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1cd0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941449.953935:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941449.953942:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0e00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941449.953943:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941449.953951:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1cd0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941449.953958:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:2.0:1773941449.953960:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941449.953962:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1cd0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941449.953970:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941449.953976:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941449.953977:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294890317 00010000:00000001:1.0:1773941449.953981:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941449.953985:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294890315 00010000:00010000:1.0:1773941449.953986:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1cd0 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941449.953993:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294890317 00010000:00000001:1.0:1773941449.953998:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:2.0:1773941449.954001:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294890315 00010000:00000001:1.0:1773941449.954003:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941449.954007:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000001:2.0:1773941449.954013:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941449.954018:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941449.954020:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1cd0 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941449.954021:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941449.954025:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1773941449.954032:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00010000:1.0:1773941449.954034:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485000 already unused, calling callback (ffffffffc1482c20) 00000100:00000001:2.0:1773941449.954035:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000800:00000010:3.0:1773941449.954038:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0e00. 00000100:00000001:2.0:1773941449.954039:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 10000000:00000001:1.0:1773941449.954040:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941449.954045:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1cd0 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941449.954047:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941449.954053:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1773941449.954058:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941449.954059:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941449.954063:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941449.954063:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:3.0:1773941449.954065:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34330 00010000:00000001:1.0:1773941449.954067:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:2.0:1773941449.954068:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941449.954069:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34330. 00000020:00000001:1.0:1773941449.954071:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941449.954074:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485000 refcount=5 00000100:00000001:3.0:1773941449.954075:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1773941449.954080:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860125696 : -128249849425920 : ffff8b5b81485000) 00000100:00000200:3.0:1773941449.954082:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60a80 x1860112241150464/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941459 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941449.954085:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860125696 : -128249849425920 : ffff8b5b81485000) 00010000:00000001:1.0:1773941449.954090:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941449.954093:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1cd0 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:1.0:1773941449.954100:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941449.954102:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1cd0 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941449.954108:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941449.954111:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941449.954112:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941449.954116:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941449.954118:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941449.954121:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941449.954123:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941449.954125:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941449.954127:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941449.954130:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485000 with handle 0xa33a5be14eae1cd0 from hash 00010000:00000001:1.0:1773941449.954137:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:3.0:1773941449.954140:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941449.954145:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60a80 x1860112241150464/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941459 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941449.954147:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941449.954150:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941449.954153:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:3.0:1773941449.954155:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941449.954157:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:3.0:1773941449.954159:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1773941449.954161:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941449.954166:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941449.954168:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941449.954169:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941449.954172:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941449.954176:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1cd0 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941449.954184:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941449.954185:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941449.954186:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941449.954188:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941449.954190:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941449.954191:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941449.954192:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941449.954194:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1cd0 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941449.954202:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962a00 count: 0 00010000:00000001:1.0:1773941449.954213:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941449.954215:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941449.954218:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941449.954224:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90277480 x1860112241150336/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941465 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941449.954231:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941449.954234:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941449.954237:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90263000. 02000000:00000001:1.0:1773941449.954241:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941449.954243:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941449.954244:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941449.954247:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941449.954249:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2a00. 02000000:00000001:1.0:1773941449.954260:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941449.954262:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941449.954264:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277480. 00000100:00000001:1.0:1773941449.954268:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941449.954269:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941449.954271:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941449.954274:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941449.954276:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941449.954280:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0cc0. 00000020:00000010:1.0:1773941449.954285:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941449.954290:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941449.954294:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941449.954298:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941449.954303:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941449.954307:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941449.954312:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941449.954317:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941449.954321:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941449.954323:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941449.954328:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941449.954332:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941449.954335:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941449.954339:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941449.954342:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941449.954346:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941449.954349:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941449.954352:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941449.954355:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0cc0. 10000000:01000000:1.0:1773941449.954357:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941449.954360:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941449.954363:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941449.954365:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941449.954366:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941449.954368:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941449.954374:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941449.955341:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941449.955346:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941449.955348:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941449.955349:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941449.955352:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941449.955356:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941449.955359:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941449.955362:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941449.955364:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941449.955366:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604a00 00000400:00000200:3.0:1773941449.955369:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941449.955372:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xcfd [1] + 0 00000800:00000001:3.0:1773941449.955375:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941449.955379:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941449.955380:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941449.955387:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941449.955444:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941449.955447:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60a80 x1860112241150464/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941459 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941449.955454:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60a80 x1860112241150464/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941459 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941449.955465:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941449.955469:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941449.955472:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941449.955492:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941449.955495:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941449.955498:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941449.955499:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941449.955501:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941449.955504:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941449.955505:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941449.955507:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941449.955512:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60a80 x1860112241150464/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941459 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941449.955520:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60a80 x1860112241150464/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941459 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941449.955525:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941449.955526:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941449.955527:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941449.955529:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941449.955530:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941449.955531:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941449.955532:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941449.955535:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=28 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941449.955537:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941449.955539:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941449.955540:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941449.955546:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941449.955547:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941449.955549:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941449.955558:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60a80 x1860112241150464/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941459 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941449.955563:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941449.955565:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941449.955566:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941449.955568:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941449.955571:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941449.955573:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941449.955574:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941449.955575:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941449.955576:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941449.955578:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941449.955580:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941449.955582:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941449 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941449.955585:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941449.955589:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941449.955590:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941449.955592:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941449.955593:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941449.955594:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941449.955596:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941449.955598:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941449.955600:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941449.955602:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941449.955604:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941449.955605:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941449.955831:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941449.955835:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941449.955841:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60a80 x1860112241150464/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941459 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941449.955846:0:12175:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941449.955851:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241150464:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00080000:1.0:1773941449.955856:0:12175:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:1.0:1773941449.955863:0:12175:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941449.955865:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941449.955866:0:12175:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941449.955870:0:12175:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941449.955871:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000001:00000001:1.0:1773941449.955873:0:12175:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941449.955876:0:12175:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941449.955879:0:12175:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941449.955881:0:12175:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941449.955882:0:12175:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941449.955884:0:12175:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941449.955885:0:12175:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941449.955886:0:12175:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941449.955889:0:12175:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941449.955891:0:12175:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941449.955893:0:12175:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000040:2.0:1773941449.955896:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60a80 x1860112241150464/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941459 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941449.955896:0:12175:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941449.955898:0:12175:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941449.955903:0:12175:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941449.955905:0:12175:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941449.955908:0:12175:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:2.0:1773941449.955916:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941449.955919:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941449.955922:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce5400. 02000000:00000001:2.0:1773941449.955927:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:1.0:1773941449.955927:0:12175:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000020:00000001:2.0:1773941449.955930:0:2351:0:(genops.c:1236:class_import_put()) Process entered 10000000:00000001:1.0:1773941449.955931:0:12175:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941449.955935:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941449.955936:0:12175:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:2.0:1773941449.955941:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:02000000:1.0:1773941449.955944:0:12175:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 02000000:00000010:2.0:1773941449.955946:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734c00. 00000100:00000001:1.0:1773941449.955951:0:12175:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941449.955952:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941449.955956:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:1.0:1773941449.955958:0:12175:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000010:2.0:1773941449.955960:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60a80. 00000020:00000040:1.0:1773941449.955962:0:12175:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941449.955966:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000020:00000001:1.0:1773941449.955967:0:12175:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941449.955969:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941449.955971:0:12175:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941449.955974:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941449.955981:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941449.955983:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941449.955985:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941449.955987:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941449.959752:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962a00. 00010000:00000010:1.1:1773941449.959760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485000. 00000020:00000001:0.0:1773941450.591917:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941450.591924:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941450.591930:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941450.975863:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941450.975874:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941450.975881:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941450.975885:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941450.975887:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941450.975889:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941450.975892:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941450.975895:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941450.975898:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941450.975901:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941450.975904:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941450.975920:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90276a00. 00000020:00000040:3.0:1773941450.975926:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941450.975933:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941450.975938:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941450.975945:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941450.975956:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e3a00. 00000100:00000001:3.0:1773941450.975966:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941450.975972:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941450.975975:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941450.975985:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962500. 00010000:00000001:3.0:1773941450.975992:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941450.975997:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81484a00. 00000020:00000001:3.0:1773941450.976002:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941450.976006:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81484a00 with handle 0xa33a5be14eae1cd7 to hash 00000020:00000001:3.0:1773941450.976008:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941450.976028:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00010000:00000001:3.0:1773941450.976032:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00010000:00000001:3.0:1773941450.976038:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941450.976042:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941450.976048:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1cd7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941450.976061:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1cd7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941450.976077:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1cd7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941450.976084:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941450.976087:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941450.976095:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941450.976098:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941450.976101:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941450.976103:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109627904 : -128249599923712 : ffff8b5b90276a00) 00000100:00000001:3.0:1773941450.976109:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941450.976115:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941450.976118:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941450.976120:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941450.976135:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90276a00 x1860112241150592/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941450.976145:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941450.976148:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941450.976153:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941450.976156:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941450.976160:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941450.976164:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241150592:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941450.976169:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941450.976186:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941450.976198:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941450.976202:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941450.976207:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941450.976210:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941450.976219:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941450.976224:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abdc0. 00000400:00000010:3.0:1773941450.976234:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a198. 00000100:00000200:3.0:1773941450.976243:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241150592, portal 25 00000100:00000001:3.0:1773941450.976246:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941450.976247:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109627904 : -128249599923712 : ffff8b5b90276a00) 00000100:00000040:3.0:1773941450.976255:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90276a00 x1860112241150592/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941466 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941450.976264:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941450.976268:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941450.976271:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a6e8. 00000100:00000200:3.0:1773941450.976275:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241150592, offset 0 00000400:00000200:3.0:1773941450.976283:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941450.976293:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941450.976303:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941450.976311:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941450.976317:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941450.976322:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941450.976325:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941450.976329:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[50, 49] 00000400:00000200:3.0:1773941450.976334:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941450.976339:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [156:156:256:4294967295] 192.168.204.153@tcp LPNI seq info [51:51:8:4294967295] 00000400:00000200:3.0:1773941450.976345:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941450.976356:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941450.976364:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941450.976370:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69500. 00000800:00000200:3.0:1773941450.976379:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941450.976395:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941450.976402:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69500 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941450.976429:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941450.976435:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941450.976439:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294891339 00000100:00000001:3.0:1773941450.976449:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294890317 00000100:00000001:3.0:1773941450.976455:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294891339 00000100:00000001:3.0:1773941450.976460:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294890317 00000100:00000001:3.0:1773941450.976464:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941450.976468:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941450.976471:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941450.976473:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941450.976478:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941450.976482:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941450.976486:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941450.976577:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941450.976585:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69500. 00000400:00000200:2.0:1773941450.976591:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941450.976599:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941450.976604:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a6e8 00000400:00000010:2.0:1773941450.976606:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a6e8. 00000100:00000001:2.0:1773941450.976611:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941450.976617:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90276a00 x1860112241150592/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941466 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941450.976627:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941450.976630:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90276a00 x1860112241150592/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941466 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941450.976650:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941450.976652:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941450.977474:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941450.977489:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941450.977493:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941450.977496:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941450.977503:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941450.977509:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941450.977568:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941450.977574:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941450.977580:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941450.977586:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604a80 00000400:00000200:3.0:1773941450.977590:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941450.977596:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xd0d [1] + 224 00000800:00000001:3.0:1773941450.977605:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941450.977614:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941450.977617:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941450.977622:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941450.977629:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941450.977641:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90276a00 x1860112241150592/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941466 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941450.977652:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90276a00 x1860112241150592/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941466 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941450.977672:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941450.977681:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941450.977685:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941450.977746:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941450.977755:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941450.977757:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941450.977761:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abdc0. 00000400:00000200:1.0:1773941450.977766:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a198 00000400:00000010:1.0:1773941450.977777:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a198. 00000100:00000001:1.0:1773941450.977782:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941450.977790:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90276a00 x1860112241150592/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941466 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941450.977802:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90276a00 x1860112241150592/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941466 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941450.977812:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941450.977814:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941450.977817:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941450.977821:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941450.977823:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941450.977825:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941450.977828:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941450.977835:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=29 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941450.977839:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941450.977843:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941450.977847:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941450.977850:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941450.977854:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941450.977857:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941450.977861:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941450.977865:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941450.977869:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941450.977872:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941450.977873:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941450.977875:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941450.977881:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90276a00 x1860112241150592/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941466 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941450.977890:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941450.977891:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941450.977893:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941450.977902:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941450.977905:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941450.977907:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941450.977909:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941450.977911:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941450.977912:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941450.977913:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941450.977915:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941450.977918:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941450.977920:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941450.977923:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941450.977927:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 220 00000400:00000200:1.0:1773941450.977934:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941450.977940:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941450.977944:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941450.977947:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941450.977950:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941450.977953:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941450.977956:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941450.977958:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941450.977961:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941450.977964:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941450.977966:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941450.977969:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941450.977973:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941450.977975:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941450.977978:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941450.977980:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941450.977984:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941450.977988:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941450.977991:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941450.977993:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941450.977997:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941450.978005:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941450.978008:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941450.978010:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941450.978016:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60e00. 00000020:00000040:1.0:1773941450.978020:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941450.978024:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941450.978026:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941450.978029:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941450.978032:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941450.978036:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941450.978043:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737c00. 00000100:00000001:1.0:1773941450.978050:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941450.978057:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60e00 x1860112241150720/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941450.978067:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941450.978072:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941450.978076:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5e4d003] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60e00 x1860112241150720/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941450.978100:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941450.978104:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941450.978106:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941450.978109:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941450.978112:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941450.978114:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941450.978117:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941450.978120:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941450.978123:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941450.978130:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90276a00 x1860112241150592/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941466 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941450.978133:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941450.978138:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941450.978140:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941450.978143:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:2.0:1773941450.978149:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60e00 x1860112241150720/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941450.978150:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941450.978156:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941450.978160:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941450.978161:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941450.978163:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941450.978166:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941450.978169:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941450.978172:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941450.978177:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90276a00 x1860112241150592/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941466 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941450.978178:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241150720:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941450.978186:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941450.978191:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:1.0:1773941450.978198:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241150592:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941450.978202:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941450.978204:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941450.978206:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941450.978210:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000010:2.0:1773941450.978213:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce4400. 00000100:00000001:1.0:1773941450.978216:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:2.0:1773941450.978217:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941450.978222:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1773941450.978226:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941450.978231:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000100:00000040:1.0:1773941450.978231:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90276a00 x1860112241150592/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941466 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941450.978238:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000200:2.0:1773941450.978244:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241150720, portal 25 00000100:00000001:1.0:1773941450.978245:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941450.978248:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941450.978251:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00000100:00000010:1.0:1773941450.978251:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:1.0:1773941450.978260:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000040:2.0:1773941450.978261:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60e00 x1860112241150720/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941460 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941450.978263:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941450.978271:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:2.0:1773941450.978272:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941450.978275:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000040:2.0:1773941450.978276:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000020:00000001:1.0:1773941450.978280:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:2.0:1773941450.978281:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34110. 00000020:00000040:1.0:1773941450.978284:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484a00 refcount=4 00000100:00000200:2.0:1773941450.978285:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241150720, offset 0 00000020:00000001:1.0:1773941450.978290:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00000400:00000200:2.0:1773941450.978294:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941450.978296:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00010000:00010000:1.0:1773941450.978302:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1cd7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941450.978309:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941450.978318:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:2.0:1773941450.978319:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941450.978326:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:1.0:1773941450.978328:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1cd7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941450.978331:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941450.978338:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941450.978343:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941450.978346:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:2.0:1773941450.978348:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[50, 51] 00000400:00000200:2.0:1773941450.978353:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00010000:1.0:1773941450.978353:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1cd7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941450.978362:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [157:157:256:4294967295] 192.168.204.143@tcp LPNI seq info [52:52:8:4294967295] 00000400:00000200:2.0:1773941450.978371:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941450.978380:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941450.978387:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941450.978390:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0600. 00000800:00000200:2.0:1773941450.978399:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941450.978406:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941450.978410:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941450.978411:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:2.0:1773941450.978416:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0600 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:1.0:1773941450.978417:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1cd7 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941450.978432:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:2.0:1773941450.978433:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941450.978439:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941450.978440:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941450.978445:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294891341 00010000:00000001:1.0:1773941450.978447:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000001:2.0:1773941450.978451:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294891339 00010000:00010000:1.0:1773941450.978453:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1cd7 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941450.978458:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294891341 00000100:00000001:2.0:1773941450.978464:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294891339 00010000:00010000:1.0:1773941450.978467:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81484a00 already unused, calling callback (ffffffffc1482c20) 00000100:00000001:2.0:1773941450.978470:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:00000001:1.0:1773941450.978472:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:2.0:1773941450.978474:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941450.978478:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00010000:1.0:1773941450.978478:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1cd7 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941450.978482:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941450.978492:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941450.978492:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:2.0:1773941450.978496:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:1.0:1773941450.978497:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:2.0:1773941450.978499:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1773941450.978501:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941450.978502:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941450.978505:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941450.978505:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484a00 refcount=5 00000020:00000001:1.0:1773941450.978510:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00000100:00000001:2.0:1773941450.978513:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941450.978516:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00010000:00000001:1.0:1773941450.978522:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941450.978525:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1cd7 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941450.978531:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 10000000:00000001:1.0:1773941450.978534:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000010:3.0:1773941450.978537:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0600. 10000000:00010000:1.0:1773941450.978541:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1cd7 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941450.978547:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:01000000:1.0:1773941450.978551:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:3.0:1773941450.978556:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941450.978562:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34110 10000000:00000040:1.0:1773941450.978564:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000010:3.0:1773941450.978567:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34110. 10000000:00000001:1.0:1773941450.978569:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941450.978576:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00010000:1.0:1773941450.978577:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941450.978581:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941450.978584:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000200:3.0:1773941450.978587:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60e00 x1860112241150720/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941460 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941450.978588:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941450.978594:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941450.978600:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941450.978604:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81484a00 with handle 0xa33a5be14eae1cd7 from hash 00010000:00000001:1.0:1773941450.978606:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941450.978608:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941450.978609:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941450.978611:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941450.978612:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941450.978614:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941450.978616:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941450.978618:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941450.978619:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941450.978621:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941450.978624:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1cd7 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941450.978630:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941450.978631:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941450.978632:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941450.978634:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941450.978636:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941450.978637:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941450.978638:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941450.978640:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1cd7 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941450.978649:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962500 count: 0 00010000:00000001:1.0:1773941450.978662:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941450.978667:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941450.978676:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941450.978681:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941450.978689:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60e00 x1860112241150720/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941460 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941450.978743:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941450.978748:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1773941450.978771:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90276a00 x1860112241150592/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941466 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941450.978782:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941450.978785:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941450.978789:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 02000000:00000001:1.0:1773941450.978793:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941450.978796:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941450.978798:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941450.978800:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941450.978803:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e3a00. 02000000:00000001:1.0:1773941450.978808:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941450.978810:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941450.978813:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90276a00. 00000100:00000001:1.0:1773941450.978816:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941450.978817:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941450.978820:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941450.978823:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941450.978825:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941450.978829:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0cc0. 00000020:00000010:1.0:1773941450.978833:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941450.978838:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941450.978841:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941450.978846:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941450.978850:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941450.978854:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941450.978859:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941450.978864:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 10000000:00000040:1.0:1773941450.978868:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941450.978870:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941450.978874:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941450.978878:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941450.978881:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941450.978885:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941450.978888:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941450.978898:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941450.978902:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941450.978904:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941450.978907:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0cc0. 10000000:01000000:1.0:1773941450.978910:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941450.978913:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941450.978915:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941450.978918:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941450.978919:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941450.978921:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941450.978923:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941450.979940:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941450.979946:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941450.979948:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941450.979950:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941450.979953:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941450.979957:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941450.979961:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941450.979963:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941450.979966:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941450.979968:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604b00 00000400:00000200:3.0:1773941450.979972:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941450.979980:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xd1d [1] + 0 00000800:00000001:3.0:1773941450.980035:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941450.980041:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941450.980042:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941450.980045:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941450.980048:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941450.980051:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60e00 x1860112241150720/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941460 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941450.980059:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60e00 x1860112241150720/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941460 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941450.980073:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941450.980077:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941450.980079:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941450.980103:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941450.980107:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941450.980110:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941450.980112:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941450.980114:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941450.980118:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941450.980119:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941450.980122:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941450.980127:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60e00 x1860112241150720/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941460 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941450.980135:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60e00 x1860112241150720/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941460 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941450.980141:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941450.980142:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941450.980144:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941450.980145:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941450.980146:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941450.980147:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941450.980149:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941450.980151:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=29 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941450.980154:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941450.980156:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941450.980157:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941450.980158:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941450.980160:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941450.980161:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941450.980164:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60e00 x1860112241150720/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941460 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941450.980169:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941450.980171:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941450.980172:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941450.980174:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941450.980178:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941450.980179:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941450.980189:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941450.980190:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941450.980192:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941450.980193:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941450.980196:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941450.980198:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941450 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941450.980201:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941450.980205:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941450.980207:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941450.980208:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941450.980209:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941450.980210:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941450.980213:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941450.980214:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941450.980216:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941450.980218:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941450.980221:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941450.980222:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941450.980453:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941450.980456:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941450.980460:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60e00 x1860112241150720/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941460 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941450.980467:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241150720:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941450.980471:0:12176:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941450.980472:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941450.980476:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941450.980481:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60e00 x1860112241150720/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941460 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941450.980487:0:12176:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:1.0:1773941450.980496:0:12176:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941450.980499:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1773941450.980501:0:12176:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941450.980503:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 10000000:01000000:1.0:1773941450.980504:0:12176:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000010:2.0:1773941450.980507:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce4400. 00000001:00000001:1.0:1773941450.980510:0:12176:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941450.980511:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941450.980516:0:12176:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:2.0:1773941450.980521:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941450.980522:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:1.0:1773941450.980523:0:12176:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941450.980526:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941450.980529:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737c00. 00000001:00000001:1.0:1773941450.980529:0:12176:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941450.980535:0:12176:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941450.980536:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941450.980539:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941450.980541:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60e00. 00000001:00000001:1.0:1773941450.980542:0:12176:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941450.980547:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 10000000:00000001:1.0:1773941450.980550:0:12176:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941450.980551:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941450.980561:0:12176:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941450.980562:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941450.980569:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941450.980572:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941450.980573:0:12176:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941450.980574:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941450.980579:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941450.980584:0:12176:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941450.980588:0:12176:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941450.980592:0:12176:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941450.980595:0:12176:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941450.980597:0:12176:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941450.980599:0:12176:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941450.980600:0:12176:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941450.980616:0:12176:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941450.980617:0:12176:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941450.980619:0:12176:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941450.980623:0:12176:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941450.980633:0:12176:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941450.980636:0:12176:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941450.980638:0:12176:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941450.980640:0:12176:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941450.980642:0:12176:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941450.983748:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962500. 00010000:00000010:1.1:1773941450.983757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81484a00. 00000020:00000001:3.0:1773941451.615902:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941451.615913:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941451.615920:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941452.127869:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941452.127880:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941452.127888:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941452.127891:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941452.127902:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941452.127904:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941452.127908:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941452.127910:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941452.127913:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941452.127917:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941452.127919:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941452.127936:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277800. 00000020:00000040:3.0:1773941452.127943:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941452.127948:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941452.127952:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941452.127957:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941452.127966:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2a00. 00000100:00000001:3.0:1773941452.127973:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941452.127998:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941452.128002:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941452.128011:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962b00. 00010000:00000001:3.0:1773941452.128018:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941452.128022:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485600. 00000020:00000001:3.0:1773941452.128027:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941452.128031:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485600 with handle 0xa33a5be14eae1cde to hash 00000020:00000001:3.0:1773941452.128034:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941452.128036:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860127232 : -128249849424384 : ffff8b5b81485600) 00010000:00000001:3.0:1773941452.128040:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860127232 : -128249849424384 : ffff8b5b81485600) 00010000:00000001:3.0:1773941452.128045:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941452.128049:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941452.128055:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1cde lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941452.128068:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1cde lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941452.128083:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1cde lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941452.128091:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941452.128094:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941452.128102:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e300. 00000100:00000001:3.0:1773941452.128105:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387400448 : -128249322151168 : ffff8b5ba0b5e300) 00000100:00000001:3.0:1773941452.128109:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941452.128110:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109631488 : -128249599920128 : ffff8b5b90277800) 00000100:00000001:3.0:1773941452.128115:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941452.128119:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941452.128121:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941452.128123:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941452.128134:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90277800 x1860112241150848/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941452.128141:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941452.128148:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941452.128152:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941452.128155:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941452.128156:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941452.128159:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241150848:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941452.128163:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941452.128169:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941452.128171:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941452.128174:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941452.128178:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90262000. 02000000:00000001:3.0:1773941452.128179:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941452.128188:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941452.128192:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abec8. 00000400:00000010:3.0:1773941452.128199:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6aaa0. 00000100:00000200:3.0:1773941452.128206:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241150848, portal 25 00000100:00000001:3.0:1773941452.128207:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941452.128209:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109631488 : -128249599920128 : ffff8b5b90277800) 00000100:00000040:3.0:1773941452.128212:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90277800 x1860112241150848/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941468 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941452.128219:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941452.128221:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941452.128222:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6add0. 00000100:00000200:3.0:1773941452.128224:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241150848, offset 0 00000400:00000200:3.0:1773941452.128230:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941452.128239:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941452.128247:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941452.128253:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941452.128257:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941452.128261:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941452.128263:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941452.128266:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[52, 51] 00000400:00000200:3.0:1773941452.128268:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941452.128272:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [158:158:256:4294967295] 192.168.204.153@tcp LPNI seq info [53:53:8:4294967295] 00000400:00000200:3.0:1773941452.128281:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941452.128291:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941452.128297:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941452.128302:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69d00. 00000800:00000200:3.0:1773941452.128309:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941452.128317:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941452.128322:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69d00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941452.128342:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941452.128345:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941452.128348:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294892491 00000100:00000001:3.0:1773941452.128355:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294891341 00000100:00000001:3.0:1773941452.128359:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294892491 00000100:00000001:3.0:1773941452.128362:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294891341 00000100:00000001:3.0:1773941452.128364:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941452.128367:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941452.128369:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941452.128371:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941452.128374:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 11 seconds 00000100:00000001:3.0:1773941452.128377:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941452.128380:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941452.128484:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941452.128489:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69d00. 00000400:00000200:2.0:1773941452.128495:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941452.128500:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941452.128505:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6add0 00000400:00000010:2.0:1773941452.128507:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6add0. 00000100:00000001:2.0:1773941452.128512:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941452.128516:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277800 x1860112241150848/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941468 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941452.128525:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941452.128528:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90277800 x1860112241150848/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941468 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941452.128534:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.128536:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941452.129580:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.129595:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941452.129598:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.129601:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941452.129605:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941452.129610:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941452.129615:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941452.129618:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941452.129622:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941452.129625:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604b80 00000400:00000200:3.0:1773941452.129629:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941452.129722:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xd2d [1] + 224 00000800:00000001:3.0:1773941452.129729:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.129737:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941452.129739:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941452.129741:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941452.129746:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941452.129750:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277800 x1860112241150848/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941468 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941452.129758:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90277800 x1860112241150848/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941468 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941452.129773:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941452.129778:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.129781:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941452.129804:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941452.129817:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941452.129819:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941452.129822:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abec8. 00000400:00000200:1.0:1773941452.129825:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6aaa0 00000400:00000010:1.0:1773941452.129827:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6aaa0. 00000100:00000001:1.0:1773941452.129830:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941452.129834:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277800 x1860112241150848/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941468 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941452.129842:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277800 x1860112241150848/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941468 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941452.129848:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941452.129849:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941452.129851:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941452.129853:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941452.129856:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941452.129858:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941452.129860:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941452.129866:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=31 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941452.129869:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941452.129872:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941452.129875:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941452.129878:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941452.129880:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941452.129883:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941452.129888:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941452.129893:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941452.129898:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941452.129908:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941452.129910:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941452.129913:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941452.129917:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90277800 x1860112241150848/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941468 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941452.129926:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941452.129928:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941452.129930:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941452.129934:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941452.129937:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941452.129939:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.129941:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941452.129943:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941452.129944:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941452.129946:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.129947:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941452.129951:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941452.129952:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941452.129955:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941452.129957:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 221 00000400:00000200:1.0:1773941452.129962:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941452.129965:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941452.129969:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941452.129972:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941452.129974:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941452.129975:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941452.129978:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941452.129980:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941452.129982:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941452.129984:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941452.129986:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941452.129988:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941452.129990:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941452.129997:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941452.129998:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941452.130000:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941452.130002:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941452.130005:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941452.130007:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941452.130009:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941452.130012:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941452.130014:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.130016:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941452.130017:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941452.130021:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61180. 00000020:00000040:1.0:1773941452.130024:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941452.130026:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941452.130028:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.130030:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941452.130032:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941452.130034:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941452.130038:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0735400. 00000100:00000001:1.0:1773941452.130041:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941452.130046:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d61180 x1860112241150976/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941452.130052:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941452.130054:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941452.130058:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e241f03b] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d61180 x1860112241150976/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941452.130074:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.130076:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941452.130077:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941452.130080:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941452.130081:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941452.130083:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941452.130088:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 1 seconds 00000100:00000001:1.0:1773941452.130089:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941452.130091:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941452.130096:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90277800 x1860112241150848/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941468 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941452.130102:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941452.130104:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941452.130106:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941452.130108:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941452.130108:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941452.130114:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1773941452.130115:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90277800 x1860112241150848/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941468 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941452.130117:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:1.0:1773941452.130127:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241150848:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000040:2.0:1773941452.130128:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d61180 x1860112241150976/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941452.130132:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941452.130136:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941452.130140:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941452.130143:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941452.130148:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1773941452.130148:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90277800 x1860112241150848/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941468 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941452.130152:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.130156:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941452.130158:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e300. 00000100:00000001:1.0:1773941452.130161:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941452.130162:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941452.130167:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:2.0:1773941452.130170:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:1.0:1773941452.130172:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:2.0:1773941452.130174:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1773941452.130176:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941452.130180:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485600 refcount=4 02000000:00000001:2.0:1773941452.130181:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941452.130186:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860127232 : -128249849424384 : ffff8b5b81485600) 00000100:00100000:2.0:1773941452.130189:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241150976:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941452.130192:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860127232 : -128249849424384 : ffff8b5b81485600) 00000100:00000001:2.0:1773941452.130194:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941452.130199:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00010000:1.0:1773941452.130200:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1cde lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941452.130201:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941452.130204:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941452.130209:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce7c00. 02000000:00000001:2.0:1773941452.130212:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941452.130215:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941452.130219:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941452.130225:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00000001:1.0:1773941452.130226:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000200:2.0:1773941452.130230:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241150976, portal 25 00010000:00010000:1.0:1773941452.130232:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1cde lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941452.130235:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941452.130238:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121074048 : -128249588477568 : ffff8b5b90d61180) 00010000:00000001:1.0:1773941452.130244:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000040:2.0:1773941452.130246:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d61180 x1860112241150976/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941462 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941452.130248:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1cde lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941452.130264:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941452.130266:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000040:2.0:1773941452.130269:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941452.130270:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941452.130273:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1cde lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941452.130275:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34000. 00000100:00000200:2.0:1773941452.130279:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241150976, offset 0 00010000:00000001:1.0:1773941452.130283:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941452.130285:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941452.130286:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941452.130290:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941452.130293:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941452.130294:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1cde lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941452.130299:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00010000:1.0:1773941452.130303:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485600 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:2.0:1773941452.130304:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00000001:1.0:1773941452.130307:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941452.130309:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 10000000:00010000:1.0:1773941452.130311:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1cde lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941452.130312:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941452.130316:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941452.130322:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[52, 53] 00000400:00000200:2.0:1773941452.130328:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941452.130330:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [159:159:256:4294967295] 192.168.204.143@tcp LPNI seq info [54:54:8:4294967295] 00010000:00000001:1.0:1773941452.130338:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:2.0:1773941452.130341:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941452.130346:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941452.130350:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941452.130351:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000020:00000040:1.0:1773941452.130356:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485600 refcount=5 00000800:00000200:2.0:1773941452.130359:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000020:00000001:1.0:1773941452.130363:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860127232 : -128249849424384 : ffff8b5b81485600) 00000800:00000010:2.0:1773941452.130365:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0500. 00010000:00000001:1.0:1773941452.130370:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860127232 : -128249849424384 : ffff8b5b81485600) 00000800:00000200:2.0:1773941452.130372:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941452.130381:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941452.130385:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0500 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941452.130386:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941452.130391:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1cde lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941452.130398:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941452.130398:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941452.130400:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1cde lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941452.130404:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941452.130408:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294892493 10000000:01000000:1.0:1773941452.130411:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:2.0:1773941452.130414:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294892491 10000000:00000040:1.0:1773941452.130415:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941452.130419:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.130422:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294892493 00010000:00010000:1.0:1773941452.130423:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941452.130427:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:2.0:1773941452.130429:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294892491 00010000:00000001:1.0:1773941452.130432:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:2.0:1773941452.130433:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941452.130434:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:2.0:1773941452.130437:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941452.130438:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:2.0:1773941452.130441:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941452.130441:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.130445:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1773941452.130446:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485600 with handle 0xa33a5be14eae1cde from hash 00010000:00000001:1.0:1773941452.130449:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941452.130450:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941452.130451:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941452.130452:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:2.0:1773941452.130453:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941452.130454:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:2.0:1773941452.130455:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:1.0:1773941452.130457:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000200:3.0:1773941452.130459:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941452.130459:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1773941452.130461:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941452.130463:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000800:00000010:3.0:1773941452.130465:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0500. 00010000:00000001:1.0:1773941452.130465:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941452.130466:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941452.130468:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.130471:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941452.130472:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000001:1.0:1773941452.130472:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941452.130478:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1cde lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941452.130481:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941452.130488:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34000 00010000:00000001:1.0:1773941452.130488:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000010:3.0:1773941452.130492:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34000. 00010000:00000001:1.0:1773941452.130492:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941452.130495:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941452.130497:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:3.0:1773941452.130498:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941452.130503:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61180 x1860112241150976/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941462 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941452.130504:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941452.130512:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000001:1.0:1773941452.130513:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000040:3.0:1773941452.130517:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d61180 x1860112241150976/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941462 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941452.130517:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941452.130525:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1cde lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941452.130530:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1773941452.130536:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962b00 count: 0 00010000:00000001:1.0:1773941452.130545:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941452.130547:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941452.130548:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941452.130553:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90277800 x1860112241150848/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941468 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941452.130558:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941452.130560:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941452.130563:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90262000. 02000000:00000001:1.0:1773941452.130565:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941452.130567:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941452.130569:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941452.130571:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941452.130573:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2a00. 02000000:00000001:1.0:1773941452.130576:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941452.130577:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941452.130579:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277800. 00000100:00000001:1.0:1773941452.130581:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941452.130582:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941452.130584:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941452.130586:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941452.130588:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941452.130592:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0e80. 00000020:00000010:1.0:1773941452.130596:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000100:00000001:3.0:1773941452.130601:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1773941452.130601:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941452.130606:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941452.130611:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941452.130615:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941452.130620:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941452.130628:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941452.130632:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941452.130636:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941452.130638:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941452.130642:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941452.130646:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941452.130648:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941452.130653:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941452.130655:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941452.130659:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941452.130662:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941452.130664:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941452.130666:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0e80. 10000000:01000000:1.0:1773941452.130668:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941452.130670:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941452.130685:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941452.130687:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941452.130688:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941452.130690:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941452.130691:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941452.131860:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.131906:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941452.131910:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.131912:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941452.131920:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941452.131937:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941452.131944:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941452.131948:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941452.131952:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941452.131955:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604c00 00000400:00000200:3.0:1773941452.131959:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941452.131964:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xd3d [1] + 0 00000800:00000001:3.0:1773941452.131970:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.131976:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941452.131978:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941452.131982:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941452.131987:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941452.131994:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61180 x1860112241150976/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941462 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941452.132004:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d61180 x1860112241150976/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941462 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941452.132021:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941452.132028:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.132031:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941452.132066:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941452.132069:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941452.132072:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941452.132073:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941452.132075:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941452.132078:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941452.132080:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941452.132082:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941452.132087:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61180 x1860112241150976/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941462 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941452.132106:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61180 x1860112241150976/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941462 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941452.132112:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941452.132113:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941452.132114:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941452.132116:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941452.132117:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941452.132119:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941452.132121:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941452.132123:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=31 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941452.132125:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941452.132128:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941452.132130:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941452.132133:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941452.132135:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.132139:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941452.132144:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d61180 x1860112241150976/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941462 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941452.132152:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941452.132155:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941452.132157:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941452.132160:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941452.132166:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941452.132169:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941452.132171:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941452.132173:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941452.132175:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941452.132177:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941452.132179:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941452.132182:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941452 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941452.132185:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941452.132191:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941452.132198:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941452.132199:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941452.132201:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.132202:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941452.132206:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941452.132208:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941452.132210:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941452.132213:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941452.132216:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941452.132219:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941452.132432:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.132436:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941452.132442:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d61180 x1860112241150976/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941462 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941452.132451:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241150976:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941452.132452:0:12178:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941452.132456:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941452.132459:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941452.132463:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d61180 x1860112241150976/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941462 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941452.132467:0:12178:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:2.0:1773941452.132474:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:1.0:1773941452.132476:0:12178:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:2.0:1773941452.132477:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941452.132480:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce7c00. 00000100:00000001:1.0:1773941452.132482:0:12178:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941452.132483:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:01000000:1.0:1773941452.132485:0:12178:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000001:2.0:1773941452.132486:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941452.132488:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941452.132491:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941452.132492:0:12178:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941452.132493:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0735400. 02000000:00000001:2.0:1773941452.132497:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:1.0:1773941452.132497:0:12178:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941452.132501:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941452.132503:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61180. 00000100:00000001:2.0:1773941452.132506:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:1.0:1773941452.132506:0:12178:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.132508:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941452.132511:0:12178:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941452.132512:0:12178:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941452.132515:0:12178:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.132517:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941452.132521:0:12178:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.132522:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941452.132524:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.132527:0:12178:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941452.132528:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941452.132531:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941452.132534:0:12178:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941452.132538:0:12178:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941452.132547:0:12178:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941452.132550:0:12178:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941452.132553:0:12178:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941452.132556:0:12178:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941452.132558:0:12178:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941452.132560:0:12178:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941452.132581:0:12178:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941452.132584:0:12178:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.132588:0:12178:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941452.132593:0:12178:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941452.132600:0:12178:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941452.132604:0:12178:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941452.132606:0:12178:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941452.132610:0:12178:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941452.132612:0:12178:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941452.135732:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962b00. 00010000:00000010:1.1:1773941452.135740:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485600. 00000020:00000001:0.0:1773941452.639912:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941452.639922:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941452.639928:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941452.703793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1773941452.703802:0:37:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1773941452.703812:0:37:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b850a8a80. 00000020:00000040:0.0:1773941452.703817:0:37:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941452.703821:0:37:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941452.703823:0:37:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941452.703829:0:37:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941452.703837:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b91e1e300. 00000100:00000001:0.0:1773941452.703843:0:37:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941452.703853:0:37:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS req@ffff8b5b850a8a80 x1860112241151104/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941452.703861:0:37:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941452.703865:0:37:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941452.703870:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d2ea9673] to pc [ptlrpcd_00_01+1] req@ffff8b5b850a8a80 x1860112241151104/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941452.703900:0:37:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941452.703904:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (228) 00000100:00000001:1.0:1773941452.703970:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941452.703977:0:2353:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941452.703980:0:2353:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941452.703992:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b850a8a80 x1860112241151104/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941452.704002:0:2353:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941452.704004:0:2353:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941452.704008:0:2353:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941452.704012:0:2353:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941452.704015:0:2353:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941452.704017:0:2353:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2353:1860112241151104:192.168.204.143@tcp:400:kworker.0 00000100:00000001:1.0:1773941452.704021:0:2353:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941452.704028:0:2353:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941452.704030:0:2353:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941452.704032:0:2353:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941452.704042:0:2353:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941452.704044:0:2353:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941452.704052:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941452.704056:0:2353:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941452.704063:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941452.704088:0:2353:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241151104, portal 25 00000100:00000001:1.0:1773941452.704090:0:2353:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941452.704092:0:2353:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615823923186304 : -128249786365312 : ffff8b5b850a8a80) 00000100:00000040:1.0:1773941452.704097:0:2353:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b850a8a80 x1860112241151104/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941452.704103:0:2353:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941452.704106:0:2353:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941452.704108:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1a18. 00000100:00000200:1.0:1773941452.704111:0:2353:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241151104, offset 0 00000400:00000200:1.0:1773941452.704116:0:2353:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941452.704124:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941452.704132:0:2353:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941452.704138:0:2353:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941452.704149:0:2353:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941452.704156:0:2353:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941452.704158:0:2353:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941452.704161:0:2353:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[54, 53] 00000400:00000200:1.0:1773941452.704164:0:2353:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941452.704168:0:2353:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [160:160:256:4294967295] 192.168.204.153@tcp LPNI seq info [55:55:8:4294967295] 00000400:00000200:1.0:1773941452.704172:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941452.704181:0:2353:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941452.704185:0:2353:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941452.704190:0:2353:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024bf00. 00000800:00000200:1.0:1773941452.704197:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941452.704209:0:2353:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941452.704213:0:2353:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024bf00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941452.704228:0:2353:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941452.704232:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941452.704234:0:2353:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294893067 00000100:00000001:1.0:1773941452.704243:0:2353:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294892493 00000100:00000001:1.0:1773941452.704255:0:2353:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294893067 00000100:00000001:1.0:1773941452.704259:0:2353:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294892493 00000100:00000001:1.0:1773941452.704262:0:2353:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941452.704265:0:2353:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.704269:0:2353:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.704272:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941452.704279:0:2353:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941452.704281:0:2353:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1773941452.704283:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941452.704284:0:2353:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941452.704286:0:2353:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.704289:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.704308:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941452.704312:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941452.704345:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941452.704351:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024bf00. 00000400:00000200:3.0:1773941452.704356:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941452.704362:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941452.704368:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1a18 00000400:00000010:3.0:1773941452.704370:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1a18. 00000100:00000001:3.0:1773941452.704373:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941452.704377:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b850a8a80 x1860112241151104/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941452.704385:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941452.704388:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b850a8a80 x1860112241151104/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941468 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941452.704393:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941452.704408:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941452.705343:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.705353:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941452.705355:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.705357:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941452.705362:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941452.705368:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941452.705373:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941452.705377:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941452.705381:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941452.705384:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604c80 00000400:00000200:3.0:1773941452.705388:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941452.705392:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xd4d [1] + 224 00000800:00000001:3.0:1773941452.705444:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.705451:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941452.705452:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941452.705455:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941452.705459:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941452.705463:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b850a8a80 x1860112241151104/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941468 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941452.705472:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b850a8a80 x1860112241151104/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941468 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941452.705485:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941452.705490:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.705492:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941452.705526:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941452.705530:0:2353:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941452.705536:0:2353:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941452.705538:0:2353:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941452.705542:0:2353:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941452.705549:0:2353:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941452.705551:0:2353:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941452.705555:0:2353:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941452.705562:0:2353:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b850a8a80 x1860112241151104/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941468 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941452.705570:0:2353:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b850a8a80 x1860112241151104/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941468 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941452.705575:0:2353:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941452.705577:0:2353:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941452.705579:0:2353:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941452.705581:0:2353:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941452.705583:0:2353:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941452.705586:0:2353:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941452.705588:0:2353:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941452.705594:0:2353:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=31 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941452.705597:0:2353:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941452.705606:0:2353:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941452.705608:0:2353:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941452.705611:0:2353:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941452.705613:0:2353:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941452.705618:0:2353:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941452.705621:0:2353:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941452.705624:0:2353:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941452.705626:0:2353:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941452.705628:0:2353:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941452.705630:0:2353:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941452.705633:0:2353:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b850a8a80 x1860112241151104/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941468 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941452.705639:0:2353:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941452.705640:0:2353:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941452.705642:0:2353:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941452.705645:0:2353:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941452.705648:0:2353:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941452.705649:0:2353:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.705650:0:2353:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941452.705651:0:2353:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941452.705652:0:2353:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941452.705652:0:2353:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.705653:0:2353:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941452.705655:0:2353:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941452.705657:0:2353:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941452.705660:0:2353:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941452.705676:0:2353:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 222 00000400:00000200:1.0:1773941452.705682:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941452.705686:0:2353:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941452.705690:0:2353:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941452.705693:0:2353:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941452.705696:0:2353:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941452.705697:0:2353:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941452.705733:0:2353:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941452.705735:0:2353:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941452.705737:0:2353:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941452.705747:0:2353:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941452.705750:0:2353:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941452.705752:0:2353:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941452.705755:0:2353:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941452.705756:0:2353:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941452.705757:0:2353:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941452.705759:0:2353:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941452.705761:0:2353:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941452.705764:0:2353:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941452.705766:0:2353:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941452.705768:0:2353:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941452.705771:0:2353:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941452.705773:0:2353:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.705775:0:2353:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941452.705777:0:2353:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941452.705781:0:2353:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61f80. 00000020:00000040:1.0:1773941452.705784:0:2353:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941452.705787:0:2353:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941452.705792:0:2353:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.705795:0:2353:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941452.705797:0:2353:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941452.705800:0:2353:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941452.705805:0:2353:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734800. 00000100:00000001:1.0:1773941452.705809:0:2353:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941452.705814:0:2353:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d61f80 x1860112241151232/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941452.705821:0:2353:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941452.705824:0:2353:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941452.705827:0:2353:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002280d596] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d61f80 x1860112241151232/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941452.705855:0:2353:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.705857:0:2353:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941452.705858:0:2353:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:1.0:1773941452.705863:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b850a8a80 x1860112241151104/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941468 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941452.705868:0:2353:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941452.705870:0:2353:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941452.705873:0:2353:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941452.705876:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b850a8a80 x1860112241151104/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941468 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941452.705881:0:2353:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2353:1860112241151104:192.168.204.143@tcp:400:kworker.0 00000100:00000001:1.0:1773941452.705884:0:2353:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941452.705886:0:2353:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941452.705888:0:2353:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b850a8a80 x1860112241151104/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941468 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941452.705893:0:2353:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941452.705896:0:2353:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941452.705898:0:2353:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941452.705901:0:2353:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941452.705903:0:2353:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941452.705905:0:2353:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941452.705906:0:2353:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941452.705909:0:2353:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b91e1e300. 02000000:00000001:1.0:1773941452.705913:0:2353:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941452.705915:0:2353:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941452.705916:0:2353:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b850a8a80. 00000100:00000001:2.0:1773941452.705918:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941452.705919:0:2353:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941452.705923:0:2353:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941452.705924:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941452.705925:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941452.705929:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941452.705930:0:2353:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941452.705937:0:2353:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941452.705939:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1773941452.705941:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d61f80 x1860112241151232/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941452.705941:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.705953:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941452.705956:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941452.705959:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941452.705961:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941452.705964:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941452.705968:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241151232:192.168.204.143@tcp:250:ptlrpcd_00_01.0 00000100:00000001:2.0:1773941452.705972:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941452.705974:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941452.705976:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941452.705978:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941452.705984:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce7c00. 02000000:00000001:2.0:1773941452.705987:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941452.705991:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941452.705995:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941452.706000:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000200:2.0:1773941452.706004:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241151232, portal 25 00000100:00000001:2.0:1773941452.706007:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941452.706008:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121077632 : -128249588473984 : ffff8b5b90d61f80) 00000100:00000040:2.0:1773941452.706014:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d61f80 x1860112241151232/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941462 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941452.706022:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941452.706025:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941452.706028:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f345d8. 00000100:00000200:2.0:1773941452.706030:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241151232, offset 0 00000400:00000200:2.0:1773941452.706044:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941452.706050:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941452.706055:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941452.706058:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941452.706061:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941452.706064:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941452.706067:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941452.706070:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[54, 55] 00000400:00000200:2.0:1773941452.706075:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941452.706079:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [161:161:256:4294967295] 192.168.204.143@tcp LPNI seq info [56:56:8:4294967295] 00000400:00000200:2.0:1773941452.706084:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941452.706092:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941452.706097:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941452.706102:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0300. 00000800:00000200:2.0:1773941452.706108:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941452.706116:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941452.706120:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0300 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941452.706149:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941452.706154:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941452.706157:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294893069 00000100:00000001:2.0:1773941452.706163:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294893067 00000100:00000001:2.0:1773941452.706169:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294893069 00000100:00000001:2.0:1773941452.706176:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294893067 00000100:00000001:2.0:1773941452.706186:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941452.706189:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.706193:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.706195:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941452.706202:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941452.706204:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941452.706206:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:3.0:1773941452.706214:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941452.706215:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941452.706219:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1773941452.706223:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0300. 00000100:00000001:2.0:1773941452.706224:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941452.706236:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941452.706244:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941452.706250:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f345d8 00000400:00000010:3.0:1773941452.706253:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f345d8. 00000100:00000001:3.0:1773941452.706257:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941452.706263:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61f80 x1860112241151232/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941462 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941452.706283:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941452.706332:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d61f80 x1860112241151232/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941462 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941452.706339:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941452.706340:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941452.707731:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.707741:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941452.707744:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.707748:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941452.707754:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941452.707799:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941452.707807:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941452.707812:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941452.707816:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941452.707820:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604d00 00000400:00000200:3.0:1773941452.707825:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941452.707829:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xd55 [1] + 0 00000800:00000001:3.0:1773941452.707849:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.707864:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941452.707866:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941452.707868:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941452.707872:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941452.707876:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61f80 x1860112241151232/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941462 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941452.707883:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d61f80 x1860112241151232/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941462 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941452.707897:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941452.707900:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941452.707902:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941452.707940:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941452.707944:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941452.707948:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941452.707950:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941452.707952:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941452.707956:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941452.707958:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941452.707961:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941452.707967:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61f80 x1860112241151232/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941462 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941452.707975:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61f80 x1860112241151232/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941462 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941452.707981:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941452.707982:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941452.707984:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941452.707985:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941452.707986:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941452.707988:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941452.707990:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941452.707992:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=31 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941452.707995:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941452.707997:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941452.707998:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941452.708001:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941452.708003:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.708006:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941452.708009:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d61f80 x1860112241151232/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941462 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941452.708014:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941452.708015:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941452.708017:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941452.708018:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941452.708033:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941452.708035:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941452.708036:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941452.708038:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941452.708040:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941452.708042:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941452.708043:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941452.708047:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941452 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941452.708049:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941452.708055:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941452.708056:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941452.708058:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941452.708059:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.708060:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941452.708063:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941452.708066:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941452.708067:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941452.708070:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941452.708073:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941452.708074:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941452.708313:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.708317:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941452.708325:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d61f80 x1860112241151232/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941462 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941452.708332:0:12179:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941452.708338:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241151232:192.168.204.143@tcp:250:ptlrpcd_00_01.0 00000100:00080000:0.0:1773941452.708346:0:12179:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941452.708358:0:12179:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941452.708359:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941452.708366:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941452.708367:0:12179:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941452.708373:0:12179:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941452.708378:0:12179:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000040:2.0:1773941452.708379:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d61f80 x1860112241151232/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941462 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000001:00000001:0.0:1773941452.708382:0:12179:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941452.708386:0:12179:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941452.708391:0:12179:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941452.708394:0:12179:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941452.708396:0:12179:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941452.708404:0:12179:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941452.708408:0:12179:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941452.708409:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00080000:0.0:1773941452.708414:0:12179:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941452.708416:0:12179:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941452.708418:0:12179:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941452.708422:0:12179:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941452.708423:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:0.0:1773941452.708425:0:12179:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941452.708428:0:12179:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:2.0:1773941452.708429:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce7c00. 10000000:01000000:0.0:1773941452.708431:0:12179:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941452.708437:0:12179:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 02000000:00000001:2.0:1773941452.708441:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941452.708445:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941452.708447:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941452.708452:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941452.708455:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734800. 00000100:00080000:0.0:1773941452.708456:0:12179:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941452.708460:0:12179:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941452.708461:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941452.708468:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941452.708470:0:12179:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000010:2.0:1773941452.708475:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61f80. 00000100:02000000:0.0:1773941452.708482:0:12179:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941452.708494:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941452.708497:0:12179:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.708501:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1773941452.708504:0:12179:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941452.708508:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1773941452.708510:0:12179:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941452.708515:0:12179:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941452.708517:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941452.708517:0:12179:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.708521:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941452.708527:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941452.708529:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941453.663917:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941453.663927:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941453.663957:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941453.855859:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941453.855872:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941453.855879:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941453.855882:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941453.855884:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941453.855886:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941453.855890:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941453.855892:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941453.855895:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941453.855899:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941453.855901:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941453.855918:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90276680. 00000020:00000040:3.0:1773941453.855925:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941453.855931:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941453.855936:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941453.855943:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941453.855953:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2600. 00000100:00000001:3.0:1773941453.855964:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941453.855969:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941453.855972:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941453.855980:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962e00. 00010000:00000001:3.0:1773941453.855986:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941453.855989:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81484400. 00000020:00000001:3.0:1773941453.855993:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941453.855996:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81484400 with handle 0xa33a5be14eae1ce5 to hash 00000020:00000001:3.0:1773941453.855998:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941453.855999:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860122624 : -128249849428992 : ffff8b5b81484400) 00010000:00000001:3.0:1773941453.856010:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860122624 : -128249849428992 : ffff8b5b81484400) 00010000:00000001:3.0:1773941453.856014:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941453.856018:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941453.856022:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1ce5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941453.856031:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1ce5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941453.856070:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1ce5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941453.856077:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941453.856080:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941453.856090:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e300. 00000100:00000001:3.0:1773941453.856092:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387400448 : -128249322151168 : ffff8b5ba0b5e300) 00000100:00000001:3.0:1773941453.856096:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941453.856097:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109627008 : -128249599924608 : ffff8b5b90276680) 00000100:00000001:3.0:1773941453.856104:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941453.856110:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941453.856112:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941453.856115:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941453.856128:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90276680 x1860112241151360/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941453.856138:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941453.856141:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941453.856145:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941453.856149:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941453.856152:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941453.856156:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241151360:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941453.856161:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941453.856169:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941453.856172:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941453.856175:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941453.856180:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941453.856183:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941453.856194:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941453.856199:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab7e8. 00000400:00000010:3.0:1773941453.856209:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6acc0. 00000100:00000200:3.0:1773941453.856218:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241151360, portal 25 00000100:00000001:3.0:1773941453.856220:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941453.856227:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109627008 : -128249599924608 : ffff8b5b90276680) 00000100:00000040:3.0:1773941453.856234:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90276680 x1860112241151360/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941469 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941453.856244:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941453.856247:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941453.856250:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6ae58. 00000100:00000200:3.0:1773941453.856254:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241151360, offset 0 00000400:00000200:3.0:1773941453.856262:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941453.856273:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941453.856283:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941453.856290:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941453.856295:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941453.856299:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941453.856302:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941453.856306:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[56, 55] 00000400:00000200:3.0:1773941453.856310:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941453.856315:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [162:162:256:4294967295] 192.168.204.153@tcp LPNI seq info [57:57:8:4294967295] 00000400:00000200:3.0:1773941453.856320:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941453.856333:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941453.856340:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941453.856346:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69d00. 00000800:00000200:3.0:1773941453.856355:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941453.856365:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941453.856371:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69d00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941453.856396:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941453.856401:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941453.856405:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294894219 00000100:00000001:3.0:1773941453.856413:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294893069 00000100:00000001:3.0:1773941453.856419:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294894219 00000100:00000001:3.0:1773941453.856423:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294893069 00000100:00000001:3.0:1773941453.856434:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941453.856438:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941453.856441:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941453.856444:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941453.856448:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 11 seconds 00000100:00000001:3.0:1773941453.856452:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941453.856456:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941453.856559:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941453.856566:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69d00. 00000400:00000200:2.0:1773941453.856573:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941453.856580:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941453.856587:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6ae58 00000400:00000010:2.0:1773941453.856590:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6ae58. 00000100:00000001:2.0:1773941453.856596:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941453.856602:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90276680 x1860112241151360/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941469 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941453.856615:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941453.856619:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90276680 x1860112241151360/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941469 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941453.856626:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941453.856629:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941453.857508:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941453.857520:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941453.857523:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941453.857525:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941453.857531:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941453.857537:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941453.857543:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941453.857548:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941453.857590:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941453.857594:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604d80 00000400:00000200:3.0:1773941453.857597:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941453.857601:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xd65 [1] + 224 00000800:00000001:3.0:1773941453.857608:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941453.857615:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941453.857616:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941453.857619:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941453.857623:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941453.857627:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90276680 x1860112241151360/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941469 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941453.857635:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90276680 x1860112241151360/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941469 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941453.857661:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941453.857667:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941453.857670:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941453.857694:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941453.857701:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941453.857725:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941453.857728:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab7e8. 00000400:00000200:1.0:1773941453.857732:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6acc0 00000400:00000010:1.0:1773941453.857734:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6acc0. 00000100:00000001:1.0:1773941453.857736:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941453.857741:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90276680 x1860112241151360/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941469 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941453.857750:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90276680 x1860112241151360/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941469 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941453.857756:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941453.857757:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941453.857759:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941453.857762:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941453.857764:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941453.857766:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941453.857768:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941453.857774:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=32 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941453.857777:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941453.857780:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941453.857783:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941453.857786:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941453.857789:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941453.857791:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941453.857795:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941453.857798:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941453.857802:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941453.857803:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941453.857804:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941453.857812:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941453.857815:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90276680 x1860112241151360/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941469 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941453.857821:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941453.857822:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941453.857824:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941453.857826:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941453.857829:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941453.857830:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941453.857831:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941453.857832:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941453.857833:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941453.857834:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941453.857836:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941453.857838:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941453.857840:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941453.857842:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941453.857844:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 223 00000400:00000200:1.0:1773941453.857847:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941453.857851:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941453.857862:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941453.857866:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941453.857868:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941453.857872:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941453.857875:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941453.857876:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941453.857878:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941453.857881:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941453.857884:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941453.857887:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941453.857891:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941453.857893:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941453.857895:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941453.857897:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941453.857908:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941453.857912:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941453.857915:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941453.857918:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941453.857922:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941453.857925:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941453.857928:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941453.857930:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941453.857935:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61500. 00000020:00000040:1.0:1773941453.857939:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941453.857943:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941453.857945:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941453.857948:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941453.857950:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941453.857953:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941453.857959:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737000. 00000100:00000001:1.0:1773941453.857963:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941453.857968:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d61500 x1860112241151488/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941453.857975:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941453.857977:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941453.857981:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000bdbed3f] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d61500 x1860112241151488/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941453.858000:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941453.858002:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941453.858004:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941453.858007:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941453.858010:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941453.858012:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941453.858014:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 1 seconds 00000100:00000001:1.0:1773941453.858016:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941453.858018:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941453.858023:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90276680 x1860112241151360/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941469 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941453.858038:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941453.858042:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941453.858045:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941453.858047:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941453.858048:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941453.858053:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941453.858058:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941453.858058:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90276680 x1860112241151360/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941469 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941453.858071:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d61500 x1860112241151488/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941453.858072:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241151360:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941453.858082:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941453.858087:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941453.858090:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941453.858092:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941453.858095:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90276680 x1860112241151360/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941469 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941453.858112:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941453.858116:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e300. 00000100:00000001:1.0:1773941453.858120:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941453.858123:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941453.858123:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941453.858128:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941453.858133:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941453.858135:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941453.858137:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:2.0:1773941453.858139:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941453.858146:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241151488:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941453.858147:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941453.858151:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941453.858153:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:1.0:1773941453.858155:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484400 refcount=4 02000000:00000001:2.0:1773941453.858158:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1773941453.858160:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860122624 : -128249849428992 : ffff8b5b81484400) 02000000:00000001:2.0:1773941453.858162:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941453.858166:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860122624 : -128249849428992 : ffff8b5b81484400) 02000000:00000001:2.0:1773941453.858167:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941453.858173:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce5800. 00010000:00010000:1.0:1773941453.858175:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1ce5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941453.858177:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941453.858187:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941453.858193:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00010000:00000001:1.0:1773941453.858194:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000010:2.0:1773941453.858201:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00010000:1.0:1773941453.858203:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1ce5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941453.858207:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241151488, portal 25 00000100:00000001:2.0:1773941453.858209:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941453.858212:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121074944 : -128249588476672 : ffff8b5b90d61500) 00010000:00000001:1.0:1773941453.858222:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000040:2.0:1773941453.858224:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d61500 x1860112241151488/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941453.858226:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1ce5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941453.858236:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941453.858244:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941453.858246:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000010:2.0:1773941453.858249:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f342a8. 00010000:00000001:1.0:1773941453.858250:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941453.858253:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1ce5 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941453.858255:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241151488, offset 0 00010000:00000001:1.0:1773941453.858265:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941453.858268:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941453.858272:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941453.858274:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1ce5 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941453.858277:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941453.858281:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81484400 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941453.858285:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941453.858288:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1ce5 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941453.858292:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941453.858298:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941453.858301:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941453.858302:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000020:00000001:1.0:1773941453.858303:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941453.858307:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484400 refcount=5 00000020:00000001:1.0:1773941453.858309:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860122624 : -128249849428992 : ffff8b5b81484400) 00000400:00000200:2.0:1773941453.858310:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941453.858312:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860122624 : -128249849428992 : ffff8b5b81484400) 00000400:00000200:2.0:1773941453.858317:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941453.858318:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941453.858322:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1ce5 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941453.858323:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941453.858331:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000001:1.0:1773941453.858333:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941453.858343:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1ce5 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941453.858347:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[56, 57] 10000000:01000000:1.0:1773941453.858355:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941453.858359:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 10000000:00000040:1.0:1773941453.858360:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941453.858364:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941453.858368:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:2.0:1773941453.858369:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [163:163:256:4294967295] 192.168.204.143@tcp LPNI seq info [58:58:8:4294967295] 00010000:00000001:1.0:1773941453.858371:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941453.858375:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941453.858378:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:2.0:1773941453.858381:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941453.858381:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941453.858383:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941453.858387:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81484400 with handle 0xa33a5be14eae1ce5 from hash 00010000:00000001:1.0:1773941453.858389:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941453.858392:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941453.858395:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941453.858397:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941453.858399:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:2.0:1773941453.858402:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941453.858403:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941453.858407:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941453.858409:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941453.858412:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941453.858414:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941453.858418:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1ce5 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941453.858419:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941453.858427:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941453.858428:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:2.0:1773941453.858430:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0300. 00010000:00000001:1.0:1773941453.858430:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941453.858433:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941453.858435:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941453.858436:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941453.858437:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941453.858439:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1ce5 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941453.858441:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000040:1.0:1773941453.858451:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962e00 count: 0 00000800:00000200:2.0:1773941453.858454:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941453.858461:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0300 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941453.858470:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941453.858473:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941453.858475:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941453.858479:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941453.858480:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90276680 x1860112241151360/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941469 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941453.858491:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941453.858492:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941453.858498:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294894221 02000000:00000001:1.0:1773941453.858498:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941453.858502:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 00000100:00000001:2.0:1773941453.858506:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294894219 02000000:00000001:1.0:1773941453.858507:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941453.858511:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941453.858514:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941453.858515:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294894221 00000020:00000001:1.0:1773941453.858516:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941453.858521:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2600. 00000100:00000001:2.0:1773941453.858523:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294894219 02000000:00000001:1.0:1773941453.858526:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941453.858529:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941453.858531:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1773941453.858531:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90276680. 00000100:00000001:1.0:1773941453.858536:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941453.858538:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941453.858539:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941453.858544:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941453.858546:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941453.858547:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:2.0:1773941453.858551:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941453.858552:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000800:00000200:3.0:1773941453.858554:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 10000000:00000010:1.0:1773941453.858557:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0e80. 00000800:00000010:3.0:1773941453.858561:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0300. 00000100:00000001:2.0:1773941453.858562:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:1.0:1773941453.858563:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000100:00000001:2.0:1773941453.858566:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000010:1.0:1773941453.858571:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000100:00000001:2.0:1773941453.858573:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941453.858577:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1773941453.858578:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000040:00000010:1.0:1773941453.858578:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000100:00000001:2.0:1773941453.858580:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941453.858586:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941453.858586:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000400:00000200:3.0:1773941453.858589:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000020:00000010:1.0:1773941453.858595:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 00000400:00000200:3.0:1773941453.858597:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f342a8 80000000:00000010:1.0:1773941453.858600:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941453.858605:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941453.858609:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 10000000:00000040:1.0:1773941453.858612:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941453.858614:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941453.858619:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941453.858622:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941453.858629:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941453.858634:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941453.858636:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941453.858654:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 00000400:00000010:3.0:1773941453.858657:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f342a8. 20000000:00000010:1.0:1773941453.858658:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941453.858664:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 00000100:00000001:3.0:1773941453.858665:0:2347:0:(events.c:34:request_out_callback()) Process entered 10000000:00000010:1.0:1773941453.858667:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0e80. 10000000:01000000:1.0:1773941453.858671:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000100:00000200:3.0:1773941453.858674:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61500 x1860112241151488/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941453.858675:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941453.858680:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941453.858683:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941453.858685:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941453.858688:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941453.858693:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:3.0:1773941453.858694:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941453.858700:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d61500 x1860112241151488/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941463 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941453.858736:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941453.858738:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941453.859914:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941453.859919:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941453.859922:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941453.859924:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941453.859928:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941453.859932:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941453.859937:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941453.859941:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941453.859944:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941453.859947:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604e00 00000400:00000200:3.0:1773941453.859951:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941453.859956:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xd75 [1] + 0 00000800:00000001:3.0:1773941453.860016:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941453.860022:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941453.860025:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941453.860029:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941453.860032:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941453.860036:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61500 x1860112241151488/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941463 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941453.860046:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d61500 x1860112241151488/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941463 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941453.860067:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941453.860073:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941453.860076:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941453.860093:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941453.860095:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941453.860099:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941453.860100:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941453.860102:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941453.860106:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941453.860115:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941453.860118:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941453.860124:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61500 x1860112241151488/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941463 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941453.860134:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61500 x1860112241151488/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941463 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941453.860142:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941453.860144:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941453.860146:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941453.860148:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941453.860150:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941453.860152:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941453.860154:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941453.860158:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=32 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941453.860166:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941453.860168:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941453.860169:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941453.860171:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941453.860173:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941453.860175:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941453.860179:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d61500 x1860112241151488/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941463 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941453.860184:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941453.860187:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941453.860189:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941453.860191:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941453.860198:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941453.860200:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941453.860201:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941453.860203:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941453.860206:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941453.860209:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941453.860211:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941453.860215:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941453 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941453.860225:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941453.860233:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941453.860235:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941453.860237:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941453.860239:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941453.860241:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941453.860246:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941453.860248:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941453.860250:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941453.860254:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941453.860256:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941453.860259:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941453.860485:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941453.860490:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941453.860496:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d61500 x1860112241151488/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941463 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941453.860501:0:12180:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:1.0:1773941453.860511:0:12180:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:1.0:1773941453.860515:0:12180:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:2.0:1773941453.860520:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241151488:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941453.860535:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941453.860537:0:12180:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941453.860539:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:01000000:1.0:1773941453.860540:0:12180:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941453.860544:0:12180:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000040:2.0:1773941453.860547:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d61500 x1860112241151488/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941463 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:1.0:1773941453.860548:0:12180:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941453.860553:0:12180:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941453.860557:0:12180:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941453.860560:0:12180:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941453.860563:0:12180:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941453.860567:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 10000000:00000001:1.0:1773941453.860568:0:12180:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941453.860572:0:12180:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:2.0:1773941453.860573:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941453.860577:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce5800. 00000100:00080000:1.0:1773941453.860577:0:12180:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941453.860583:0:12180:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:2.0:1773941453.860585:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941453.860586:0:12180:0:(recover.c:147:ptlrpc_resend()) Process entered 00000020:00000001:2.0:1773941453.860590:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941453.860591:0:12180:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941453.860595:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00080000:1.0:1773941453.860596:0:12180:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000020:00000001:2.0:1773941453.860603:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941453.860603:0:12180:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941453.860605:0:12180:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941453.860608:0:12180:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 02000000:00000010:2.0:1773941453.860609:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737000. 02000000:00000001:2.0:1773941453.860614:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941453.860616:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941453.860618:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61500. 00000100:00080000:1.0:1773941453.860625:0:12180:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:2.0:1773941453.860630:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 10000000:00000001:1.0:1773941453.860634:0:12180:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941453.860635:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941453.860638:0:12180:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941453.860652:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:02000000:1.0:1773941453.860656:0:12180:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941453.860661:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941453.860662:0:12180:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941453.860665:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941453.860666:0:12180:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941453.860669:0:12180:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941453.860671:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1773941453.860673:0:12180:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941453.860675:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941453.860676:0:12180:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941453.863742:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962e00. 00010000:00000010:1.1:1773941453.863751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81484400. 00000020:00000001:0.0:1773941454.687912:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941454.687921:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941454.687928:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941455.327881:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941455.327892:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941455.327899:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941455.327902:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941455.327904:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941455.327906:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941455.327910:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941455.327912:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941455.327915:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941455.327919:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941455.327922:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941455.327935:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62300. 00000020:00000040:1.0:1773941455.327941:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941455.327947:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941455.327952:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941455.327959:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941455.327969:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941455.327977:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941455.327983:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941455.327986:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941455.327995:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec500. 00010000:00000001:1.0:1773941455.328002:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941455.328005:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138e600. 00000020:00000001:1.0:1773941455.328009:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941455.328012:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138e600 with handle 0xa33a5be14eae1cec to hash 00000020:00000001:1.0:1773941455.328014:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941455.328015:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941455.328018:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941455.328037:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941455.328040:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941455.328044:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cec lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941455.328053:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cec lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941455.328064:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cec lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941455.328069:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941455.328071:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941455.328077:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941455.328079:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941455.328082:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941455.328083:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121078528 : -128249588473088 : ffff8b5b90d62300) 00000100:00000001:1.0:1773941455.328088:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941455.328092:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941455.328094:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941455.328096:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941455.328107:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62300 x1860112241151616/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941455.328114:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941455.328116:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941455.328119:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941455.328121:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941455.328123:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941455.328126:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241151616:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941455.328129:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941455.328134:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941455.328136:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941455.328139:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941455.328146:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734000. 02000000:00000001:1.0:1773941455.328148:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941455.328156:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941455.328159:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941455.328165:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941455.328173:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241151616, portal 25 00000100:00000001:1.0:1773941455.328174:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941455.328175:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121078528 : -128249588473088 : ffff8b5b90d62300) 00000100:00000040:1.0:1773941455.328179:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62300 x1860112241151616/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941471 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941455.328186:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941455.328188:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941455.328190:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f15d8. 00000100:00000200:1.0:1773941455.328192:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241151616, offset 0 00000400:00000200:1.0:1773941455.328198:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941455.328207:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941455.328217:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941455.328224:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941455.328230:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941455.328235:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941455.328238:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941455.328242:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[58, 57] 00000400:00000200:1.0:1773941455.328247:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941455.328251:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [164:164:256:4294967295] 192.168.204.153@tcp LPNI seq info [59:59:8:4294967295] 00000400:00000200:1.0:1773941455.328256:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941455.328269:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941455.328277:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941455.328284:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b400. 00000800:00000200:1.0:1773941455.328292:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941455.328302:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941455.328308:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b400 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941455.328335:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941455.328341:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941455.328345:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294895691 00000100:00000001:1.0:1773941455.328353:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294894221 00000100:00000001:1.0:1773941455.328359:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294895691 00000100:00000001:1.0:1773941455.328365:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294894221 00000100:00000001:1.0:1773941455.328369:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941455.328372:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941455.328374:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941455.328377:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941455.328380:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941455.328384:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941455.328387:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941455.328462:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941455.328467:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b400. 00000400:00000200:3.0:1773941455.328472:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941455.328478:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941455.328483:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f15d8 00000400:00000010:3.0:1773941455.328485:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f15d8. 00000100:00000001:3.0:1773941455.328489:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941455.328493:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62300 x1860112241151616/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941471 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941455.328502:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941455.328505:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d62300 x1860112241151616/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941471 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941455.328510:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941455.328512:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941455.329390:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941455.329404:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941455.329407:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941455.329409:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941455.329462:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941455.329469:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941455.329476:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941455.329481:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941455.329485:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941455.329488:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604e80 00000400:00000200:3.0:1773941455.329492:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941455.329497:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xd85 [1] + 224 00000800:00000001:3.0:1773941455.329503:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941455.329509:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941455.329511:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941455.329527:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941455.329533:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941455.329540:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62300 x1860112241151616/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941471 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941455.329549:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d62300 x1860112241151616/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941471 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941455.329565:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941455.329572:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941455.329575:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941455.329591:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941455.329599:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941455.329601:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941455.329604:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941455.329607:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941455.329609:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941455.329611:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941455.329628:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62300 x1860112241151616/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941471 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941455.329636:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62300 x1860112241151616/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941471 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941455.329642:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941455.329643:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941455.329645:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941455.329647:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941455.329650:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941455.329652:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941455.329654:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941455.329660:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=34 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941455.329663:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941455.329665:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941455.329675:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941455.329677:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941455.329680:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941455.329683:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941455.329687:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941455.329690:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941455.329693:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941455.329695:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941455.329696:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941455.329698:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941455.329700:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d62300 x1860112241151616/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941471 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941455.329729:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941455.329730:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941455.329731:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941455.329734:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941455.329737:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941455.329738:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941455.329739:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941455.329740:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941455.329741:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941455.329742:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941455.329743:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941455.329745:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941455.329746:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941455.329748:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941455.329750:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 224 00000400:00000200:1.0:1773941455.329754:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941455.329758:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941455.329761:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941455.329764:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941455.329766:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941455.329775:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941455.329777:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941455.329779:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941455.329781:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941455.329783:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941455.329785:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941455.329786:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941455.329788:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941455.329789:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941455.329790:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941455.329791:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941455.329793:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941455.329796:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941455.329798:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941455.329800:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941455.329803:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941455.329805:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941455.329807:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941455.329808:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941455.329812:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61880. 00000020:00000040:1.0:1773941455.329814:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941455.329816:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941455.329818:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941455.329820:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941455.329821:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941455.329823:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941455.329826:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737400. 00000100:00000001:1.0:1773941455.329829:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941455.329841:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d61880 x1860112241151744/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941455.329850:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941455.329852:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941455.329860:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d153cd32] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d61880 x1860112241151744/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941455.329875:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941455.329877:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941455.329878:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941455.329880:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941455.329883:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941455.329884:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941455.329885:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941455.329887:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941455.329889:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941455.329892:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62300 x1860112241151616/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941471 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941455.329898:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941455.329900:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941455.329903:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941455.329904:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941455.329908:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62300 x1860112241151616/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941471 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941455.329914:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241151616:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941455.329917:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941455.329918:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941455.329923:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941455.329924:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941455.329926:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941455.329928:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941455.329930:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941455.329938:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62300 x1860112241151616/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941471 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941455.329941:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d61880 x1860112241151744/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941455.329957:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941455.329964:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000010:1.0:1773941455.329964:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:2.0:1773941455.329968:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941455.329970:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941455.329973:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941455.329982:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:1.0:1773941455.329983:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:2.0:1773941455.329986:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941455.329989:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:2.0:1773941455.329992:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941455.329993:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941455.329995:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138e600 refcount=4 00000020:00000001:1.0:1773941455.329998:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00000100:00100000:2.0:1773941455.330001:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241151744:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941455.330002:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00010000:1.0:1773941455.330007:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cec lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941455.330011:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941455.330016:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:1.0:1773941455.330018:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:2.0:1773941455.330019:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941455.330024:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00010000:1.0:1773941455.330024:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cec lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941455.330032:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000010:2.0:1773941455.330033:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce5800. 00010000:00010000:1.0:1773941455.330036:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cec lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941455.330038:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941455.330044:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941455.330048:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941455.330048:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941455.330053:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cec lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941455.330056:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00010000:00000001:1.0:1773941455.330060:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941455.330062:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1773941455.330064:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00000001:1.0:1773941455.330065:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941455.330068:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cec lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941455.330072:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241151744, portal 25 00010000:00010000:1.0:1773941455.330074:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138e600 already unused, calling callback (ffffffffc1482c20) 00000100:00000001:2.0:1773941455.330077:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 10000000:00000001:1.0:1773941455.330078:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:2.0:1773941455.330081:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121075840 : -128249588475776 : ffff8b5b90d61880) 10000000:00010000:1.0:1773941455.330082:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cec lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941455.330092:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d61880 x1860112241151744/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941465 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941455.330111:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:2.0:1773941455.330114:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941455.330115:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000040:2.0:1773941455.330118:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000020:00000001:1.0:1773941455.330120:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941455.330122:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138e600 refcount=5 00000020:00000001:1.0:1773941455.330125:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941455.330127:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00000400:00000010:2.0:1773941455.330133:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34b28. 00010000:00000001:1.0:1773941455.330134:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000200:2.0:1773941455.330136:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241151744, offset 0 00010000:00010000:1.0:1773941455.330140:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cec lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941455.330144:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941455.330152:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941455.330158:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 10000000:00000001:1.0:1773941455.330158:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941455.330164:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cec lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941455.330165:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941455.330171:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941455.330176:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:01000000:1.0:1773941455.330179:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941455.330182:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000040:1.0:1773941455.330185:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941455.330188:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941455.330191:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[58, 59] 00010000:00010000:1.0:1773941455.330192:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941455.330196:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941455.330200:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941455.330201:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941455.330205:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941455.330207:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941455.330209:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [165:165:256:4294967295] 192.168.204.143@tcp LPNI seq info [60:60:8:4294967295] 00010000:00000001:1.0:1773941455.330210:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941455.330216:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138e600 with handle 0xa33a5be14eae1cec from hash 00000400:00000200:2.0:1773941455.330220:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941455.330220:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941455.330223:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941455.330225:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941455.330227:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941455.330230:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:2.0:1773941455.330233:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941455.330234:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941455.330239:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941455.330243:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941455.330243:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941455.330247:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1773941455.330250:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0500. 00000800:00000200:2.0:1773941455.330256:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00000001:1.0:1773941455.330256:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941455.330262:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cec lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941455.330268:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941455.330273:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941455.330274:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0500 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941455.330276:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941455.330279:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941455.330280:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941455.330281:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941455.330282:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941455.330283:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941455.330285:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1cec lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941455.330288:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1773941455.330290:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec500 count: 0 00000400:00000200:2.0:1773941455.330294:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941455.330297:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294895693 00000100:00000001:2.0:1773941455.330302:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294895691 00000100:00000001:2.0:1773941455.330306:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294895693 00010000:00000001:1.0:1773941455.330307:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941455.330311:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294895691 00010000:00000001:1.0:1773941455.330311:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941455.330316:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941455.330317:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941455.330320:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941455.330324:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941455.330324:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62300 x1860112241151616/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941471 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941455.330327:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941455.330334:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941455.330336:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:1.0:1773941455.330337:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941455.330340:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734000. 02000000:00000001:1.0:1773941455.330342:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1773941455.330346:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000001:1.0:1773941455.330347:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941455.330351:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941455.330353:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1773941455.330354:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941455.330357:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 02000000:00000010:1.0:1773941455.330359:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 00000100:00000001:2.0:1773941455.330360:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941455.330364:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941455.330365:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941455.330367:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941455.330369:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62300. 00000100:00000001:1.0:1773941455.330372:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941455.330372:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941455.330374:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941455.330376:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941455.330379:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941455.330388:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0e80. 00000020:00000010:1.0:1773941455.330392:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941455.330398:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941455.330403:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941455.330407:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941455.330411:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941455.330416:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000800:00000200:3.0:1773941455.330418:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00040000:00000010:1.0:1773941455.330422:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 00000800:00000010:3.0:1773941455.330423:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0500. 20000000:00000010:1.0:1773941455.330429:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 00000400:00000200:3.0:1773941455.330432:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000040:1.0:1773941455.330435:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941455.330439:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:3.0:1773941455.330443:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000020:00000010:1.0:1773941455.330450:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000400:00000200:3.0:1773941455.330451:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34b28 00000040:00000010:1.0:1773941455.330457:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941455.330462:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941455.330466:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941455.330468:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941455.330472:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941455.330474:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941455.330477:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941455.330479:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0e80. 10000000:01000000:1.0:1773941455.330481:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941455.330484:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941455.330485:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941455.330487:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941455.330489:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941455.330490:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941455.330492:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000400:00000010:3.0:1773941455.330498:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34b28. 00000100:00000001:3.0:1773941455.330505:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941455.330510:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61880 x1860112241151744/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941465 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941455.330518:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941455.330520:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d61880 x1860112241151744/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941465 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941455.330526:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941455.330527:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941455.331667:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941455.331674:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941455.331676:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941455.331677:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941455.331681:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941455.331684:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941455.331771:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941455.331774:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941455.331779:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941455.331792:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321604f00 00000400:00000200:3.0:1773941455.331796:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941455.331802:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xd8d [1] + 0 00000800:00000001:3.0:1773941455.331808:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941455.331816:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941455.331819:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941455.331823:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941455.331827:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941455.331833:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61880 x1860112241151744/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941465 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941455.331844:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d61880 x1860112241151744/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941465 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941455.331860:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941455.331864:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941455.331867:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941455.331891:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941455.331894:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941455.331897:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941455.331898:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941455.331900:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941455.331903:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941455.331905:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941455.331907:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941455.331913:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61880 x1860112241151744/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941465 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941455.331920:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61880 x1860112241151744/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941465 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941455.331925:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941455.331927:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941455.331928:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941455.331929:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941455.331931:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941455.331932:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941455.331934:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941455.331936:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=34 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941455.331939:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941455.331940:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941455.331942:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941455.331944:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941455.331945:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941455.331947:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941455.331950:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d61880 x1860112241151744/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941465 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941455.331955:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941455.331957:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941455.331964:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941455.331967:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941455.331973:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941455.331976:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941455.331977:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941455.331979:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941455.331981:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941455.331984:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941455.331987:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941455.331991:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941455 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941455.331995:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941455.332002:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941455.332005:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941455.332007:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941455.332009:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941455.332011:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941455.332015:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941455.332018:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941455.332021:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941455.332025:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941455.332028:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941455.332031:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941455.332237:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941455.332240:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941455.332245:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d61880 x1860112241151744/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941465 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941455.332253:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241151744:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941455.332254:0:12181:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941455.332258:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941455.332261:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1773941455.332267:0:12181:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000040:2.0:1773941455.332268:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d61880 x1860112241151744/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941465 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941455.332275:0:12181:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941455.332278:0:12181:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941455.332279:0:12181:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941455.332282:0:12181:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941455.332286:0:12181:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941455.332288:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941455.332290:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941455.332292:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce5800. 02000000:00000001:2.0:1773941455.332297:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941455.332298:0:12181:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941455.332299:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941455.332302:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:1.0:1773941455.332303:0:12181:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941455.332306:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941455.332307:0:12181:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941455.332310:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737400. 00000001:00000001:1.0:1773941455.332311:0:12181:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941455.332314:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000001:1.0:1773941455.332314:0:12181:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941455.332317:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941455.332318:0:12181:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000010:2.0:1773941455.332320:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61880. 00000100:00080000:1.0:1773941455.332323:0:12181:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941455.332324:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941455.332326:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941455.332328:0:12181:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941455.332329:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941455.332332:0:12181:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941455.332335:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941455.332335:0:12181:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941455.332338:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941455.332340:0:12181:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941455.332342:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941455.332344:0:12181:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941455.332345:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941455.332346:0:12181:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941455.332349:0:12181:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941455.332367:0:12181:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941455.332368:0:12181:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941455.332370:0:12181:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941455.332373:0:12181:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941455.332376:0:12181:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941455.332378:0:12181:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941455.332380:0:12181:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941455.332382:0:12181:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941455.332384:0:12181:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941455.335742:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec500. 00010000:00000010:1.1:1773941455.335750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138e600. 00000020:00000001:3.0:1773941455.711863:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941455.711871:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941455.711876:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1773941456.735869:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941456.735875:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941456.735879:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941457.247820:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941457.247827:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941457.247831:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941457.247833:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941457.247834:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941457.247836:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941457.247838:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941457.247840:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941457.247842:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941457.247844:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941457.247846:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941457.247856:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90274380. 00000020:00000040:3.0:1773941457.247860:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941457.247864:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941457.247867:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941457.247872:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941457.247893:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2a00. 00000100:00000001:3.0:1773941457.247899:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941457.247903:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941457.247905:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941457.247912:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962d00. 00010000:00000001:3.0:1773941457.247917:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941457.247920:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81484c00. 00000020:00000001:3.0:1773941457.247923:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941457.247925:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81484c00 with handle 0xa33a5be14eae1cf3 to hash 00000020:00000001:3.0:1773941457.247927:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941457.247928:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860124672 : -128249849426944 : ffff8b5b81484c00) 00010000:00000001:3.0:1773941457.247931:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860124672 : -128249849426944 : ffff8b5b81484c00) 00010000:00000001:3.0:1773941457.247933:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941457.247936:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941457.247940:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1cf3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941457.247951:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1cf3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941457.247963:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1cf3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941457.247970:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941457.247972:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941457.247979:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941457.247982:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941457.247985:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941457.247986:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109618048 : -128249599933568 : ffff8b5b90274380) 00000100:00000001:3.0:1773941457.247991:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941457.247996:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941457.247999:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941457.248000:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941457.248011:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90274380 x1860112241151872/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941457.248022:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941457.248024:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941457.248027:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941457.248029:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941457.248031:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941457.248034:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90274380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241151872:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941457.248037:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941457.248041:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941457.248043:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941457.248045:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941457.248048:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90263000. 02000000:00000001:3.0:1773941457.248050:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941457.248056:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941457.248059:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab9a0. 00000400:00000010:3.0:1773941457.248077:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941457.248082:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241151872, portal 25 00000100:00000001:3.0:1773941457.248084:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941457.248085:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109618048 : -128249599933568 : ffff8b5b90274380) 00000100:00000040:3.0:1773941457.248089:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90274380 x1860112241151872/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941473 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941457.248095:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941457.248097:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941457.248099:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a3b8. 00000100:00000200:3.0:1773941457.248101:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241151872, offset 0 00000400:00000200:3.0:1773941457.248105:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941457.248112:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941457.248118:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941457.248122:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941457.248125:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941457.248128:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941457.248131:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941457.248137:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[60, 59] 00000400:00000200:3.0:1773941457.248140:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941457.248143:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [166:166:256:4294967295] 192.168.204.153@tcp LPNI seq info [61:61:8:4294967295] 00000400:00000200:3.0:1773941457.248146:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941457.248154:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941457.248159:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941457.248162:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69500. 00000800:00000200:3.0:1773941457.248167:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941457.248173:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941457.248177:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69500 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941457.248194:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941457.248197:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941457.248199:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294897611 00000100:00000001:3.0:1773941457.248205:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294895693 00000100:00000001:3.0:1773941457.248210:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294897611 00000100:00000001:3.0:1773941457.248213:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294895693 00000100:00000001:3.0:1773941457.248215:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941457.248218:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941457.248220:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941457.248221:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941457.248223:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941457.248226:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941457.248228:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941457.248336:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941457.248343:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69500. 00000400:00000200:2.0:1773941457.248349:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941457.248356:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941457.248361:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a3b8 00000400:00000010:2.0:1773941457.248364:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a3b8. 00000100:00000001:2.0:1773941457.248369:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941457.248375:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90274380 x1860112241151872/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941473 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941457.248386:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941457.248390:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90274380 x1860112241151872/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941473 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941457.248398:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941457.248400:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941457.249126:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.249139:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941457.249141:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.249144:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941457.249149:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941457.249154:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941457.249161:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941457.249164:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941457.249223:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941457.249226:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321604f80 00000400:00000200:3.0:1773941457.249229:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941457.249233:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xd9d [1] + 224 00000800:00000001:3.0:1773941457.249238:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.249244:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941457.249246:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941457.249248:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941457.249251:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941457.249255:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90274380 x1860112241151872/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941473 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941457.249263:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90274380 x1860112241151872/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941473 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941457.249275:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941457.249281:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.249283:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941457.249304:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941457.249310:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941457.249312:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941457.249314:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab9a0. 00000400:00000200:1.0:1773941457.249318:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:1.0:1773941457.249319:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:1.0:1773941457.249322:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941457.249326:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90274380 x1860112241151872/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941473 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941457.249335:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90274380 x1860112241151872/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941473 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941457.249340:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941457.249347:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941457.249349:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941457.249352:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941457.249354:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941457.249355:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941457.249357:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941457.249363:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=36 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941457.249365:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941457.249368:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941457.249371:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941457.254981:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941457.254990:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941457.254997:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941457.255003:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941457.255009:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941457.263278:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941457.263283:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941457.263286:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941457.263288:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941457.263298:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90274380 x1860112241151872/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941473 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941457.263311:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941457.263312:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941457.263315:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941457.263318:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941457.263321:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941457.263323:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.263325:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941457.263327:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941457.263328:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941457.263330:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.263332:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941457.263334:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941457.263338:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941457.263356:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941457.263360:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 225 00000400:00000200:1.0:1773941457.263369:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941457.263375:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941457.263380:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941457.263383:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941457.263386:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941457.263389:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941457.263393:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941457.263395:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941457.263398:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941457.263401:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941457.263404:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941457.263407:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941457.263411:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941457.263413:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941457.263416:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941457.263419:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941457.263422:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941457.263426:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941457.263429:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941457.263432:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941457.263436:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941457.263439:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.263442:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941457.263444:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941457.263450:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62300. 00000020:00000040:1.0:1773941457.263453:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941457.263457:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941457.263460:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.263463:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941457.263466:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941457.263476:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941457.263484:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734000. 00000100:00000001:1.0:1773941457.263491:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941457.263498:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d62300 x1860112241152000/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941457.263510:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941457.263514:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941457.263519:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000000582022] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d62300 x1860112241152000/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941457.263541:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.263544:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941457.263546:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941457.263550:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941457.263554:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941457.263556:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941457.263560:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941457.263563:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941457.263566:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941457.263566:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941457.263572:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1773941457.263589:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90274380 x1860112241151872/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941473 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941457.263594:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941457.263601:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941457.263606:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941457.263609:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:2.0:1773941457.263612:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62300 x1860112241152000/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941457.263613:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941457.263623:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90274380 x1860112241151872/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941473 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941457.263629:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941457.263634:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941457.263640:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:1.0:1773941457.263642:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90274380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241151872:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941457.263644:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941457.263650:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.263653:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941457.263658:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241152000:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941457.263659:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941457.263664:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941457.263666:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941457.263670:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941457.263675:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.263675:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941457.263680:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1773941457.263684:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90274380 x1860112241151872/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941473 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1773941457.263688:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce5000. 02000000:00000001:2.0:1773941457.263693:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.263701:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941457.263733:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941457.263738:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941457.263745:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000010:1.0:1773941457.263749:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000200:2.0:1773941457.263753:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241152000, portal 25 00000100:00000001:1.0:1773941457.263757:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941457.263758:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941457.263762:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121078528 : -128249588473088 : ffff8b5b90d62300) 00000100:00000001:1.0:1773941457.263762:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941457.263770:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000040:2.0:1773941457.263775:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62300 x1860112241152000/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941457.263776:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941457.263780:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941457.263784:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484c00 refcount=4 00000100:00000001:2.0:1773941457.263788:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1773941457.263790:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860124672 : -128249849426944 : ffff8b5b81484c00) 00000100:00000040:2.0:1773941457.263796:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941457.263797:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860124672 : -128249849426944 : ffff8b5b81484c00) 00000400:00000010:2.0:1773941457.263800:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34908. 00000100:00000200:2.0:1773941457.263804:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241152000, offset 0 00010000:00010000:1.0:1773941457.263805:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1cf3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941457.263813:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941457.263821:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:2.0:1773941457.263826:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941457.263830:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1cf3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941457.263835:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941457.263844:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941457.263844:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941457.263850:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1cf3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941457.263853:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941457.263860:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941457.263862:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941457.263865:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941457.263867:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1cf3 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941457.263871:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941457.263880:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[60, 61] 00010000:00000001:1.0:1773941457.263882:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941457.263887:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941457.263887:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941457.263893:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [167:167:256:4294967295] 192.168.204.143@tcp LPNI seq info [62:62:8:4294967295] 00010000:00000001:1.0:1773941457.263894:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941457.263899:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941457.263912:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00010000:1.0:1773941457.263912:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1cf3 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941457.263927:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941457.263929:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81484c00 already unused, calling callback (ffffffffc1482c20) 00000800:00000010:2.0:1773941457.263934:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0600. 10000000:00000001:1.0:1773941457.263934:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941457.263940:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1cf3 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941457.263947:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941457.263954:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941457.263957:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941457.263960:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:2.0:1773941457.263961:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000020:00000040:1.0:1773941457.263964:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484c00 refcount=5 00000800:00000200:2.0:1773941457.263967:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0600 type 1, nob 616 niov 1 nkiov 1 00000020:00000001:1.0:1773941457.263968:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860124672 : -128249849426944 : ffff8b5b81484c00) 00010000:00000001:1.0:1773941457.263974:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860124672 : -128249849426944 : ffff8b5b81484c00) 00010000:00000001:1.0:1773941457.263977:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941457.263980:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1cf3 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941457.263985:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941457.263989:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941457.263994:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00010000:1.0:1773941457.263995:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1cf3 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941457.263998:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294897627 10000000:01000000:1.0:1773941457.264006:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:2.0:1773941457.264009:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294897611 10000000:00000040:1.0:1773941457.264011:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941457.264015:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941457.264019:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294897627 00010000:00010000:1.0:1773941457.264020:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941457.264024:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:2.0:1773941457.264026:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294897611 00010000:00000001:1.0:1773941457.264029:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:2.0:1773941457.264031:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941457.264033:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:2.0:1773941457.264037:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941457.264038:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:2.0:1773941457.264043:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941457.264043:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941457.264050:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1773941457.264051:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81484c00 with handle 0xa33a5be14eae1cf3 from hash 00010000:00000001:1.0:1773941457.264057:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:2.0:1773941457.264061:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941457.264064:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941457.264068:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1773941457.264068:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941457.264071:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941457.264072:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941457.264075:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:2.0:1773941457.264076:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941457.264079:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:2.0:1773941457.264080:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941457.264083:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000200:3.0:1773941457.264088:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941457.264089:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941457.264092:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:3.0:1773941457.264095:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0600. 00010000:00000001:1.0:1773941457.264096:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941457.264101:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000001:1.0:1773941457.264101:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941457.264107:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1cf3 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941457.264109:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941457.264116:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34908 00000400:00000010:3.0:1773941457.264120:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34908. 00010000:00000001:1.0:1773941457.264121:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941457.264124:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941457.264125:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941457.264128:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000200:3.0:1773941457.264130:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62300 x1860112241152000/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941457.264131:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941457.264135:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941457.264138:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941457.264141:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941457.264145:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00010000:1.0:1773941457.264146:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1cf3 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941457.264150:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62300 x1860112241152000/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941467 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941457.264167:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1773941457.264168:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962d00 count: 0 00000100:00000001:3.0:1773941457.264169:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1773941457.264187:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941457.264190:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941457.264194:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941457.264200:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90274380 x1860112241151872/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941473 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941457.264208:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941457.264211:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941457.264214:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90263000. 02000000:00000001:1.0:1773941457.264218:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941457.264220:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941457.264222:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941457.264225:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941457.264227:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2a00. 02000000:00000001:1.0:1773941457.264237:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941457.264239:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941457.264241:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90274380. 00000100:00000001:1.0:1773941457.264244:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941457.264246:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941457.264248:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941457.264251:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941457.264255:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941457.264259:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0e80. 00000020:00000010:1.0:1773941457.264263:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941457.264268:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941457.264272:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941457.264277:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941457.264282:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941457.264286:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941457.264292:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941457.264297:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 10000000:00000040:1.0:1773941457.264301:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941457.264303:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941457.264308:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941457.264313:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941457.264316:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941457.264321:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941457.264324:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941457.264329:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941457.264331:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941457.264334:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941457.264337:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0e80. 10000000:01000000:1.0:1773941457.264340:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941457.264342:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941457.264345:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941457.264348:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941457.264350:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941457.264352:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941457.264360:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:1.1:1773941457.269748:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962d00. 00010000:00000010:1.1:1773941457.269762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81484c00. 00000800:00000001:3.0:1773941457.270897:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.270908:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941457.270923:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.270925:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941457.270994:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941457.271000:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941457.271006:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941457.271010:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941457.271014:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941457.271017:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605000 00000400:00000200:3.0:1773941457.271021:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941457.271026:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xdad [1] + 0 00000800:00000001:3.0:1773941457.271032:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.271038:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941457.271040:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941457.271043:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941457.271048:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941457.271054:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62300 x1860112241152000/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941467 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941457.271071:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d62300 x1860112241152000/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941467 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941457.271084:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941457.271091:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.271095:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941457.271120:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941457.271124:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941457.271130:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941457.271132:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941457.271136:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941457.271146:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941457.271155:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941457.271158:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941457.271162:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62300 x1860112241152000/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941467 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941457.271171:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62300 x1860112241152000/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941467 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941457.271176:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941457.271178:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941457.271180:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941457.271183:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941457.271185:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941457.271187:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941457.271189:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941457.271194:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=36 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941457.271197:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941457.271199:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941457.271200:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941457.271202:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941457.271204:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941457.271206:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941457.271209:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62300 x1860112241152000/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941467 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941457.271214:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941457.271216:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941457.271218:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941457.271220:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941457.271227:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941457.271230:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941457.271232:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941457.271234:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941457.271237:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941457.271240:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941457.271242:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941457.271255:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941457 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941457.271259:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941457.271265:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941457.271268:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941457.271271:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941457.271274:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941457.271276:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941457.271279:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941457.271282:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941457.271285:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941457.271289:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941457.271292:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941457.271294:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941457.271496:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941457.271499:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941457.271504:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62300 x1860112241152000/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941467 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941457.271511:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241152000:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941457.271516:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941457.271518:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941457.271520:0:12182:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941457.271522:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62300 x1860112241152000/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941467 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941457.271531:0:12182:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:1.0:1773941457.271536:0:12182:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941457.271538:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1773941457.271539:0:12182:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941457.271542:0:12182:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941457.271543:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941457.271545:0:12182:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941457.271549:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce5000. 00000001:00000001:1.0:1773941457.271549:0:12182:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941457.271553:0:12182:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941457.271555:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941457.271556:0:12182:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941457.271559:0:12182:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:2.0:1773941457.271561:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:1.0:1773941457.271561:0:12182:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941457.271565:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 10000000:00000001:1.0:1773941457.271566:0:12182:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.271569:0:12182:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:2.0:1773941457.271573:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:1.0:1773941457.271585:0:12182:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:2.0:1773941457.271591:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734000. 00000100:00000001:1.0:1773941457.271592:0:12182:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941457.271596:0:12182:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:2.0:1773941457.271598:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941457.271600:0:12182:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941457.271604:0:12182:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 02000000:00000001:2.0:1773941457.271605:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941457.271609:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62300. 00000100:00000001:1.0:1773941457.271610:0:12182:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941457.271613:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 10000000:01000000:1.0:1773941457.271614:0:12182:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:2.0:1773941457.271616:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000040:1.0:1773941457.271617:0:12182:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:2.0:1773941457.271620:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941457.271628:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941457.271630:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941457.271632:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941457.271634:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941457.271637:0:12182:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941457.271640:0:12182:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.271641:0:12182:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941457.271645:0:12182:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941457.275124:0:12182:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941457.275127:0:12182:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941457.275128:0:12182:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941457.275130:0:12182:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941457.275131:0:12182:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941457.759942:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941457.759949:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941457.759952:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941457.823862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1773941457.823874:0:37:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1773941457.823884:0:37:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62300. 00000020:00000040:1.0:1773941457.823889:0:37:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941457.823907:0:37:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941457.823909:0:37:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941457.823914:0:37:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941457.823920:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b9024bb00. 00000100:00000001:1.0:1773941457.823926:0:37:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941457.823935:0:37:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS req@ffff8b5b90d62300 x1860112241152128/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941457.823944:0:37:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941457.823947:0:37:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941457.823952:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000000582022] to pc [ptlrpcd_00_00+0] req@ffff8b5b90d62300 x1860112241152128/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941457.823980:0:37:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941457.823983:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (233) 00000100:00000001:1.0:1773941457.823998:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941457.823999:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1773941457.824001:0:2353:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1773941457.824003:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.824005:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941457.824012:0:2353:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941457.824015:0:2353:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941457.824016:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941457.824018:0:2353:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941457.824020:0:2353:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941457.824024:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62300 x1860112241152128/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941457.824029:0:2353:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941457.824031:0:2353:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941457.824034:0:2353:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941457.824036:0:2353:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941457.824038:0:2353:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941457.824040:0:2353:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2353:1860112241152128:192.168.204.143@tcp:400:kworker.0 00000100:00000001:1.0:1773941457.824053:0:2353:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941457.824062:0:2353:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941457.824065:0:2353:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941457.824068:0:2353:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941457.824079:0:2353:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941457.824082:0:2353:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941457.824090:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941457.824095:0:2353:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941457.824109:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941457.824117:0:2353:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241152128, portal 25 00000100:00000001:1.0:1773941457.824123:0:2353:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941457.824125:0:2353:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121078528 : -128249588473088 : ffff8b5b90d62300) 00000100:00000040:1.0:1773941457.824131:0:2353:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62300 x1860112241152128/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941457.824137:0:2353:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941457.824139:0:2353:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941457.824141:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1b28. 00000100:00000200:1.0:1773941457.824144:0:2353:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241152128, offset 0 00000400:00000200:1.0:1773941457.824149:0:2353:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941457.824156:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941457.824162:0:2353:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941457.824166:0:2353:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941457.824169:0:2353:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941457.824177:0:2353:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941457.824178:0:2353:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941457.824181:0:2353:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[62, 61] 00000400:00000200:1.0:1773941457.824183:0:2353:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941457.824188:0:2353:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [168:168:256:4294967295] 192.168.204.153@tcp LPNI seq info [63:63:8:4294967295] 00000400:00000200:1.0:1773941457.824192:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941457.824204:0:2353:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941457.824211:0:2353:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941457.824216:0:2353:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b600. 00000800:00000200:1.0:1773941457.824223:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941457.824231:0:2353:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941457.824236:0:2353:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b600 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941457.824259:0:2353:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941457.824264:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941457.824266:0:2353:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294898187 00000100:00000001:1.0:1773941457.824271:0:2353:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294897627 00000100:00000001:1.0:1773941457.824274:0:2353:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294898187 00000100:00000001:1.0:1773941457.824279:0:2353:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294897627 00000100:00000001:1.0:1773941457.824282:0:2353:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941457.824284:0:2353:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.824287:0:2353:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.824289:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941457.824363:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941457.824367:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b600. 00000400:00000200:3.0:1773941457.824372:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941457.824377:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941457.824382:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1b28 00000400:00000010:3.0:1773941457.824383:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1b28. 00000100:00000001:3.0:1773941457.824386:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941457.824391:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62300 x1860112241152128/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941457.824404:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941457.824407:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62300 x1860112241152128/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941473 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941457.824412:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941457.824414:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941457.825735:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.825754:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941457.825756:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.825759:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941457.825765:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941457.825770:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941457.825843:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941457.825848:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941457.825852:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941457.825858:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605080 00000400:00000200:3.0:1773941457.825862:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941457.825867:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xdbd [1] + 224 00000800:00000001:3.0:1773941457.825874:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.825882:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941457.825885:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941457.825889:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941457.825893:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941457.825899:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62300 x1860112241152128/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941473 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941457.825910:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b90d62300 x1860112241152128/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941473 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941457.825930:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941457.825937:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.825940:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941457.825966:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941457.825969:0:2353:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941457.825973:0:2353:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941457.825975:0:2353:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941457.825977:0:2353:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941457.825980:0:2353:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941457.825982:0:2353:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941457.825984:0:2353:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941457.825988:0:2353:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62300 x1860112241152128/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941473 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941457.825996:0:2353:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62300 x1860112241152128/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941473 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941457.826001:0:2353:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941457.826002:0:2353:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941457.826004:0:2353:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941457.826006:0:2353:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941457.826008:0:2353:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941457.826009:0:2353:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941457.826011:0:2353:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941457.826017:0:2353:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=36 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941457.826019:0:2353:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941457.826022:0:2353:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941457.826023:0:2353:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941457.826025:0:2353:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941457.826034:0:2353:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941457.826038:0:2353:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941457.826041:0:2353:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941457.826044:0:2353:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941457.826046:0:2353:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941457.826047:0:2353:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941457.826049:0:2353:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941457.826051:0:2353:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d62300 x1860112241152128/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941473 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941457.826057:0:2353:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941457.826058:0:2353:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941457.826059:0:2353:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941457.826061:0:2353:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941457.826063:0:2353:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941457.826064:0:2353:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.826065:0:2353:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941457.826066:0:2353:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941457.826067:0:2353:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941457.826068:0:2353:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.826069:0:2353:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941457.826072:0:2353:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941457.826074:0:2353:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941457.826077:0:2353:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941457.826080:0:2353:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 227 00000400:00000200:1.0:1773941457.826084:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941457.826089:0:2353:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941457.826093:0:2353:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941457.826096:0:2353:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941457.826098:0:2353:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941457.826101:0:2353:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941457.826105:0:2353:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941457.826107:0:2353:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941457.826110:0:2353:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941457.826118:0:2353:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941457.826121:0:2353:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941457.826124:0:2353:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941457.826128:0:2353:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941457.826130:0:2353:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941457.826132:0:2353:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941457.826134:0:2353:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941457.826136:0:2353:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941457.826139:0:2353:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941457.826141:0:2353:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941457.826142:0:2353:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941457.826145:0:2353:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941457.826156:0:2353:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.826159:0:2353:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941457.826161:0:2353:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941457.826166:0:2353:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61880. 00000020:00000040:1.0:1773941457.826170:0:2353:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941457.826173:0:2353:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941457.826175:0:2353:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.826178:0:2353:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941457.826180:0:2353:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941457.826183:0:2353:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941457.826189:0:2353:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0736800. 00000100:00000001:1.0:1773941457.826193:0:2353:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941457.826199:0:2353:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d61880 x1860112241152256/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941457.826207:0:2353:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941457.826210:0:2353:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941457.826214:0:2353:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d153cd32] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d61880 x1860112241152256/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941457.826235:0:2353:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.826238:0:2353:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941457.826246:0:2353:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:1.0:1773941457.826257:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62300 x1860112241152128/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941473 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941457.826266:0:2353:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941457.826270:0:2353:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941457.826271:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941457.826273:0:2353:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941457.826275:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941457.826278:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941457.826282:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62300 x1860112241152128/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941473 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941457.826286:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d61880 x1860112241152256/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941457.826307:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941457.826308:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941457.826312:0:2353:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2353:1860112241152128:192.168.204.143@tcp:400:kworker.0 02000000:00000001:2.0:1773941457.826313:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941457.826316:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941457.826319:0:2353:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941457.826320:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.826326:0:2353:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1773941457.826329:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241152256:192.168.204.143@tcp:250:ptlrpcd_00_01.0 00000100:00000040:1.0:1773941457.826333:0:2353:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62300 x1860112241152128/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941473 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941457.826334:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941457.826337:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941457.826340:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941457.826344:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941457.826346:0:2353:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000010:2.0:1773941457.826349:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce5000. 02000000:00000001:1.0:1773941457.826350:0:2353:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1773941457.826354:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941457.826354:0:2353:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941457.826358:0:2353:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:2.0:1773941457.826360:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:1.0:1773941457.826361:0:2353:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000010:2.0:1773941457.826364:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000020:00000040:1.0:1773941457.826365:0:2353:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941457.826370:0:2353:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000010:2.0:1773941457.826372:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 02000000:00000010:1.0:1773941457.826373:0:2353:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b9024bb00. 00000100:00000200:2.0:1773941457.826378:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241152256, portal 25 02000000:00000001:1.0:1773941457.826378:0:2353:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941457.826381:0:2353:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941457.826382:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000010:1.0:1773941457.826384:0:2353:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62300. 00000100:00000001:2.0:1773941457.826385:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121075840 : -128249588475776 : ffff8b5b90d61880) 00000100:00000001:1.0:1773941457.826387:0:2353:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941457.826390:0:2353:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941457.826393:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d61880 x1860112241152256/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941457.826394:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941457.826403:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1773941457.826403:0:2353:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941457.826407:0:2353:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.826410:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1773941457.826414:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000100:00000001:1.0:1773941457.826415:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1773941457.826417:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34440. 00000100:00000200:2.0:1773941457.826420:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241152256, offset 0 00000400:00000200:2.0:1773941457.826424:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941457.826428:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941457.826432:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941457.826434:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941457.826437:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941457.826439:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941457.826440:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941457.826443:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[62, 63] 00000400:00000200:2.0:1773941457.826445:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941457.826448:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [169:169:256:4294967295] 192.168.204.143@tcp LPNI seq info [64:64:8:4294967295] 00000400:00000200:2.0:1773941457.826451:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941457.826456:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941457.826459:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941457.826462:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0e00. 00000800:00000200:2.0:1773941457.826466:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941457.826471:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941457.826474:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0e00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941457.826486:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941457.826489:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941457.826490:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294898189 00000100:00000001:2.0:1773941457.826494:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294898187 00000100:00000001:2.0:1773941457.826497:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294898189 00000100:00000001:2.0:1773941457.826499:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294898187 00000100:00000001:2.0:1773941457.826501:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941457.826503:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941457.826505:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941457.826507:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941457.826514:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941457.826516:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941457.826517:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941457.826519:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941457.826520:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941457.826521:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941457.826550:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941457.826553:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0e00. 00000400:00000200:3.0:1773941457.826562:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941457.826576:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941457.826579:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34440 00000400:00000010:3.0:1773941457.826580:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34440. 00000100:00000001:3.0:1773941457.826583:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941457.826587:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61880 x1860112241152256/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941467 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941457.826595:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941457.826599:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d61880 x1860112241152256/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941467 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941457.826605:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941457.826607:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941457.827716:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.827721:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941457.827723:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.827725:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941457.827728:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941457.827731:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941457.827735:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941457.827738:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941457.827746:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941457.827749:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605100 00000400:00000200:3.0:1773941457.827752:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941457.827754:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xdc5 [1] + 0 00000800:00000001:3.0:1773941457.827758:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.827812:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941457.827814:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941457.827817:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941457.827820:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941457.827823:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61880 x1860112241152256/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941467 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941457.827829:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d61880 x1860112241152256/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941467 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941457.827847:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941457.827852:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941457.827855:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941457.827888:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941457.827891:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941457.827895:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941457.827896:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941457.827898:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941457.827902:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941457.827904:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941457.827907:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941457.827912:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61880 x1860112241152256/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941467 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941457.827923:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61880 x1860112241152256/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941467 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941457.827931:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941457.827933:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941457.827936:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941457.827938:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941457.827939:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941457.827941:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941457.827944:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941457.827948:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=36 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941457.827951:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941457.827954:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941457.827955:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941457.827958:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941457.827961:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941457.827963:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941457.827967:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d61880 x1860112241152256/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941467 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941457.827975:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941457.827977:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941457.827985:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941457.827987:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941457.827994:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941457.827997:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941457.827998:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941457.828000:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941457.828002:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941457.828005:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941457.828007:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941457.828011:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941457 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941457.828014:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941457.828020:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941457.828023:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941457.828024:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941457.828026:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941457.828028:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941457.828032:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941457.828034:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941457.828036:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941457.828039:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941457.828042:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941457.828045:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941457.828288:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941457.828293:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941457.828299:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d61880 x1860112241152256/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941467 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941457.828309:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d61880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241152256:192.168.204.143@tcp:250:ptlrpcd_00_01.0 00000100:00000001:1.0:1773941457.828314:0:12183:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941457.828315:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941457.828318:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941457.828323:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d61880 x1860112241152256/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941467 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941457.828329:0:12183:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:1.0:1773941457.828335:0:12183:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941457.828338:0:12183:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941457.828339:0:12183:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941457.828342:0:12183:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941457.828344:0:12183:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941457.828348:0:12183:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941457.828352:0:12183:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941457.828354:0:12183:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941457.828356:0:12183:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941457.828360:0:12183:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941457.828361:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1773941457.828362:0:12183:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:2.0:1773941457.828367:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:1.0:1773941457.828368:0:12183:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:2.0:1773941457.828373:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce5000. 00000100:00000001:1.0:1773941457.828373:0:12183:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941457.828376:0:12183:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941457.828379:0:12183:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941457.828380:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:1.0:1773941457.828381:0:12183:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000020:00000001:2.0:1773941457.828384:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941457.828385:0:12183:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941457.828388:0:12183:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000020:00000040:2.0:1773941457.828389:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 10000000:00000040:1.0:1773941457.828390:0:12183:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000020:00000001:2.0:1773941457.828395:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941457.828396:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0736800. 02000000:00000001:2.0:1773941457.828399:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941457.828400:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941457.828401:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61880. 00000100:00080000:1.0:1773941457.828404:0:12183:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:2.0:1773941457.828405:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941457.828408:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941457.828408:0:12183:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.828413:0:12183:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941457.828414:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:02000000:1.0:1773941457.828419:0:12183:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941457.828420:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941457.828423:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941457.828423:0:12183:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941457.828429:0:12183:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941457.828430:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1773941457.828432:0:12183:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941457.828433:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941457.828436:0:12183:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941457.828439:0:12183:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941458.719828:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941458.719837:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941458.719842:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941458.719845:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941458.719847:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941458.719848:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941458.719851:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941458.719853:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941458.719855:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941458.719858:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941458.719860:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941458.719871:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90276d80. 00000020:00000040:3.0:1773941458.719876:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941458.719881:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941458.719884:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941458.719889:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941458.719897:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e3000. 00000100:00000001:3.0:1773941458.719903:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941458.719917:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941458.719920:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941458.719928:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962400. 00010000:00000001:3.0:1773941458.719934:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941458.719938:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485400. 00000020:00000001:3.0:1773941458.719942:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941458.719945:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485400 with handle 0xa33a5be14eae1cfa to hash 00000020:00000001:3.0:1773941458.719948:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941458.719949:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860126720 : -128249849424896 : ffff8b5b81485400) 00010000:00000001:3.0:1773941458.719953:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860126720 : -128249849424896 : ffff8b5b81485400) 00010000:00000001:3.0:1773941458.719956:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941458.719960:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941458.719965:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1cfa lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941458.719990:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1cfa lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941458.720003:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1cfa lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941458.720009:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941458.720011:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941458.720018:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e9c0. 00000100:00000001:3.0:1773941458.720020:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387402176 : -128249322149440 : ffff8b5ba0b5e9c0) 00000100:00000001:3.0:1773941458.720023:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941458.720025:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109628800 : -128249599922816 : ffff8b5b90276d80) 00000100:00000001:3.0:1773941458.720030:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941458.720034:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941458.720037:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941458.720039:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941458.720049:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90276d80 x1860112241152384/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941458.720058:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941458.720060:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941458.720064:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941458.720066:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941458.720069:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941458.720073:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241152384:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941458.720077:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941458.720082:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941458.720084:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941458.720087:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941458.720091:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90263000. 02000000:00000001:3.0:1773941458.720093:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941458.720100:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941458.720110:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab3c8. 00000400:00000010:3.0:1773941458.720117:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a110. 00000100:00000200:3.0:1773941458.720123:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241152384, portal 25 00000100:00000001:3.0:1773941458.720126:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941458.720127:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109628800 : -128249599922816 : ffff8b5b90276d80) 00000100:00000040:3.0:1773941458.720133:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90276d80 x1860112241152384/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941474 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941458.720140:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941458.720143:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941458.720146:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a330. 00000100:00000200:3.0:1773941458.720149:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241152384, offset 0 00000400:00000200:3.0:1773941458.720155:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941458.720163:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941458.720170:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941458.720176:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941458.720180:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941458.720185:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941458.720188:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941458.720191:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[64, 63] 00000400:00000200:3.0:1773941458.720195:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941458.720200:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [170:170:256:4294967295] 192.168.204.153@tcp LPNI seq info [65:65:8:4294967295] 00000400:00000200:3.0:1773941458.720204:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941458.720214:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941458.720221:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941458.720225:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69f00. 00000800:00000200:3.0:1773941458.720232:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941458.720239:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941458.720244:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69f00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941458.720266:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941458.720270:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941458.720273:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294899083 00000100:00000001:3.0:1773941458.720285:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294898189 00000100:00000001:3.0:1773941458.720291:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294899083 00000100:00000001:3.0:1773941458.720294:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294898189 00000100:00000001:3.0:1773941458.720298:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941458.720301:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941458.720304:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941458.720306:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941458.720309:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941458.720313:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941458.720316:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941458.720398:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941458.720404:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69f00. 00000400:00000200:2.0:1773941458.720409:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941458.720415:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941458.720420:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a330 00000400:00000010:2.0:1773941458.720423:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a330. 00000100:00000001:2.0:1773941458.720427:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941458.720432:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90276d80 x1860112241152384/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941474 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941458.720443:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941458.720447:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90276d80 x1860112241152384/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941474 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941458.720454:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941458.720456:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941458.721607:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941458.721619:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941458.721622:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941458.721624:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941458.721630:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941458.721638:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941458.721644:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941458.721654:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941458.721658:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941458.721740:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605180 00000400:00000200:3.0:1773941458.721747:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941458.721754:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xdd5 [1] + 224 00000800:00000001:3.0:1773941458.721761:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941458.721770:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941458.721773:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941458.721777:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941458.721782:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941458.721787:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90276d80 x1860112241152384/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941474 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941458.721805:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90276d80 x1860112241152384/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941474 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941458.721836:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941458.721852:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941458.721857:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941458.721875:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941458.721882:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941458.721884:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941458.721887:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab3c8. 00000400:00000200:1.0:1773941458.721892:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a110 00000400:00000010:1.0:1773941458.721894:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a110. 00000100:00000001:1.0:1773941458.721898:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941458.721909:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90276d80 x1860112241152384/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941474 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941458.721921:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90276d80 x1860112241152384/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941474 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941458.721928:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941458.721930:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941458.721933:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941458.721937:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941458.721939:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941458.721941:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941458.721944:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941458.721949:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=37 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941458.721953:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941458.721955:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941458.721960:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941458.721963:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941458.721966:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941458.721969:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941458.721973:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941458.721978:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941458.721981:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941458.721983:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941458.721985:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941458.721987:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941458.721991:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90276d80 x1860112241152384/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941474 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941458.721999:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941458.722000:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941458.722002:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941458.722005:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941458.722008:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941458.722010:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941458.722016:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941458.722017:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941458.722019:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941458.722020:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941458.722022:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941458.722025:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941458.722027:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941458.722030:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941458.722033:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 228 00000400:00000200:1.0:1773941458.722038:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941458.722042:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941458.722046:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941458.722049:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941458.722052:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941458.722055:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941458.722058:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941458.722060:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941458.722062:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941458.722064:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941458.722067:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941458.722069:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941458.722072:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941458.722074:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941458.722076:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941458.722078:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941458.722081:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941458.722084:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941458.722087:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941458.722090:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941458.722093:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941458.722096:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941458.722098:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941458.722105:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941458.722110:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62300. 00000020:00000040:1.0:1773941458.722113:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941458.722117:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941458.722118:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941458.722121:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941458.722123:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941458.722125:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941458.722130:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734000. 00000100:00000001:1.0:1773941458.722134:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941458.722147:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d62300 x1860112241152512/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941458.722156:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941458.722159:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941458.722163:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000000582022] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d62300 x1860112241152512/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941458.722186:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941458.722190:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941458.722198:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941458.722201:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941458.722205:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941458.722207:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941458.722209:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941458.722213:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941458.722215:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941458.722221:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90276d80 x1860112241152384/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941474 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941458.722226:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941458.722231:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941458.722232:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941458.722235:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941458.722237:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941458.722240:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941458.722242:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941458.722247:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90276d80 x1860112241152384/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941474 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941458.722252:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62300 x1860112241152512/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941458.722268:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941458.722271:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941458.722277:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941458.722280:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941458.722285:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941458.722290:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241152384:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1773941458.722293:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241152512:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941458.722298:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941458.722300:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941458.722303:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941458.722305:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941458.722308:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:2.0:1773941458.722309:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941458.722313:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941458.722314:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941458.722320:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce5000. 00000100:00000040:1.0:1773941458.722321:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90276d80 x1860112241152384/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941474 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941458.722325:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941458.722337:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941458.722338:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1773941458.722342:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000100:00000010:1.0:1773941458.722342:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e9c0. 00000100:00000001:1.0:1773941458.722349:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000400:00000010:2.0:1773941458.722351:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000001:1.0:1773941458.722351:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941458.722357:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000200:2.0:1773941458.722358:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241152512, portal 25 00010000:00000001:1.0:1773941458.722360:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:2.0:1773941458.722362:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000020:00000001:1.0:1773941458.722364:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941458.722366:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121078528 : -128249588473088 : ffff8b5b90d62300) 00000020:00000040:1.0:1773941458.722368:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485400 refcount=4 00000020:00000001:1.0:1773941458.722373:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860126720 : -128249849424896 : ffff8b5b81485400) 00000100:00000040:2.0:1773941458.722376:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62300 x1860112241152512/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941458.722377:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860126720 : -128249849424896 : ffff8b5b81485400) 00010000:00010000:1.0:1773941458.722386:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1cfa lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941458.722390:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941458.722396:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941458.722400:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34c38. 00010000:00000001:1.0:1773941458.722402:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000200:2.0:1773941458.722406:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241152512, offset 0 00010000:00010000:1.0:1773941458.722409:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1cfa lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941458.722416:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941458.722422:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:2.0:1773941458.722423:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941458.722427:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1cfa lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941458.722431:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941458.722439:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941458.722442:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941458.722445:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941458.722449:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941458.722453:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941458.722457:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1cfa lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941458.722465:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941458.722471:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941458.722472:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[64, 65] 00010000:00000001:1.0:1773941458.722476:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941458.722479:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941458.722481:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941458.722486:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [171:171:256:4294967295] 192.168.204.143@tcp LPNI seq info [66:66:8:4294967295] 00010000:00010000:1.0:1773941458.722487:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1cfa lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941458.722497:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941458.722505:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485400 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941458.722510:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941458.722513:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 10000000:00010000:1.0:1773941458.722515:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1cfa lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941458.722524:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941458.722532:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0800. 00010000:00000001:1.0:1773941458.722532:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941458.722536:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941458.722539:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941458.722542:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485400 refcount=5 00000800:00000200:2.0:1773941458.722543:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000020:00000001:1.0:1773941458.722546:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860126720 : -128249849424896 : ffff8b5b81485400) 00010000:00000001:1.0:1773941458.722564:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860126720 : -128249849424896 : ffff8b5b81485400) 00000800:00000200:2.0:1773941458.722569:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941458.722570:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000200:2.0:1773941458.722576:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0800 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:1.0:1773941458.722577:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1cfa lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:1.0:1773941458.722587:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941458.722589:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1cfa lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941458.722595:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941458.722596:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941458.722602:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000040:1.0:1773941458.722603:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941458.722607:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941458.722608:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294899085 00010000:00010000:1.0:1773941458.722611:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:2.0:1773941458.722615:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294899083 00010000:00000001:1.0:1773941458.722615:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:2.0:1773941458.722621:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294899085 00010000:00000001:1.0:1773941458.722626:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:2.0:1773941458.722627:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294899083 00010000:00000001:1.0:1773941458.722630:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941458.722634:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:2.0:1773941458.722635:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941458.722637:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941458.722639:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941458.722642:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485400 with handle 0xa33a5be14eae1cfa from hash 00000100:00000001:2.0:1773941458.722644:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941458.722647:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:2.0:1773941458.722649:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941458.722651:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941458.722654:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941458.722655:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941458.722657:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:2.0:1773941458.722658:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941458.722659:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:2.0:1773941458.722662:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:1.0:1773941458.722665:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941458.722666:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1773941458.722668:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941458.722670:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941458.722672:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941458.722673:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941458.722676:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941458.722678:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941458.722683:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1cfa lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941458.722688:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941458.722691:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000010:3.0:1773941458.722694:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0800. 00010000:00000001:1.0:1773941458.722694:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941458.722698:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941458.722701:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000400:00000200:3.0:1773941458.722725:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941458.722732:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941458.722737:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34c38 00000400:00000010:3.0:1773941458.722740:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34c38. 00010000:00000001:1.0:1773941458.722743:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941458.722745:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941458.722747:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941458.722750:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000200:3.0:1773941458.722753:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62300 x1860112241152512/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941458.722754:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1cfa lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941458.722764:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962400 count: 0 00010000:00000001:1.0:1773941458.722774:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941458.722776:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941458.722779:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941458.722785:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90276d80 x1860112241152384/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941474 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941458.722793:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941458.722795:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941458.722803:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90263000. 02000000:00000001:1.0:1773941458.722809:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941458.722813:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941458.722816:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941458.722819:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1773941458.722820:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941458.722825:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e3000. 00000100:00000040:3.0:1773941458.722828:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62300 x1860112241152512/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941468 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941458.722830:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941458.722834:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941458.722837:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90276d80. 00000100:00000001:1.0:1773941458.722843:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941458.722844:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941458.722847:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941458.722848:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000001:1.0:1773941458.722851:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941458.722855:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941458.722858:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941458.722861:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0e80. 00000020:00000010:1.0:1773941458.722865:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941458.722870:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941458.722874:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941458.722879:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941458.722883:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941458.722887:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941458.722892:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941458.722896:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941458.722900:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941458.722902:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941458.722907:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941458.722910:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941458.722913:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941458.722917:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941458.722919:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941458.722923:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941458.722926:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941458.722928:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941458.722931:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0e80. 10000000:01000000:1.0:1773941458.722933:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941458.722940:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941458.722942:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000800:00000001:3.0:1773941458.724307:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941458.724315:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941458.724318:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941458.724321:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941458.724326:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941458.724331:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941458.724337:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941458.724341:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941458.724344:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941458.724348:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605200 00000400:00000200:3.0:1773941458.724352:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941458.724419:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xde5 [1] + 0 00000800:00000001:3.0:1773941458.724427:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941458.724434:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941458.724436:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941458.724448:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941458.724453:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941458.724458:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62300 x1860112241152512/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941468 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941458.724469:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d62300 x1860112241152512/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941468 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941458.724488:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941458.724493:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941458.724498:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941458.724536:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941458.724540:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941458.724545:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941458.724546:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941458.724564:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941458.724574:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941458.724577:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941458.724580:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941458.724586:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62300 x1860112241152512/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941468 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941458.724596:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62300 x1860112241152512/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941468 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941458.724604:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941458.724606:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941458.724609:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941458.724611:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941458.724613:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941458.724614:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941458.724617:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941458.724621:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=37 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941458.724625:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941458.724627:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941458.724629:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941458.724631:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941458.724633:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941458.724637:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941458.724641:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62300 x1860112241152512/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941468 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941458.724648:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941458.724650:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941458.724652:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941458.724655:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941458.724660:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941458.724662:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941458.724663:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941458.724666:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941458.724668:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941458.724670:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941458.724673:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941458.724683:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941458 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941458.724686:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941458.724693:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941458.724695:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941458.724698:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941458.724700:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941458.724702:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941458.724726:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941458.724728:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941458.724731:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941458.724734:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941458.724738:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941458.724741:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941458.724956:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941458.724961:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941458.724976:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62300 x1860112241152512/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941468 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941458.724977:0:12184:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:2.0:1773941458.724986:0:12184:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:2.0:1773941458.724990:0:12184:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:0.0:1773941458.724992:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241152512:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941458.725002:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941458.725005:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941458.725010:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62300 x1860112241152512/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941468 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941458.725012:0:12184:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941458.725015:0:12184:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941458.725019:0:12184:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941458.725023:0:12184:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:0.0:1773941458.725025:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:2.0:1773941458.725027:0:12184:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941458.725029:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:2.0:1773941458.725030:0:12184:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941458.725034:0:12184:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:0.0:1773941458.725035:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce5000. 00000001:00000001:2.0:1773941458.725036:0:12184:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941458.725039:0:12184:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941458.725043:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941458.725046:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941458.725047:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941458.725049:0:12184:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:0.0:1773941458.725050:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:2.0:1773941458.725055:0:12184:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:0.0:1773941458.725055:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734000. 02000000:00000001:0.0:1773941458.725059:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941458.725062:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941458.725063:0:12184:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941458.725066:0:12184:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000010:0.0:1773941458.725066:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62300. 00000100:00000001:2.0:1773941458.725071:0:12184:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941458.725073:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:2.0:1773941458.725074:0:12184:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941458.725077:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941458.725078:0:12184:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941458.725080:0:12184:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:0.0:1773941458.725083:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000040:2.0:1773941458.725084:0:12184:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:0.0:1773941458.725091:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941458.725094:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941458.725096:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941458.725098:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941458.725102:0:12184:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941458.725104:0:12184:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941458.725105:0:12184:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941458.725108:0:12184:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941458.725111:0:12184:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941458.725112:0:12184:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941458.725114:0:12184:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941458.725115:0:12184:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941458.725117:0:12184:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941458.728662:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941458.728666:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941458.728668:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941458.728671:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:1.1:1773941458.733785:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962400. 00010000:00000010:1.1:1773941458.733794:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485400. 00000020:00000001:0.0:1773941458.783898:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941458.783905:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941458.783910:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941459.807873:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941459.807879:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941459.807882:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941460.767815:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941460.767824:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941460.767828:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941460.767831:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941460.767833:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941460.767834:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941460.767837:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941460.767839:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941460.767842:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941460.767845:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941460.767846:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941460.767859:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61500. 00000020:00000040:1.0:1773941460.767866:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941460.767872:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941460.767876:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941460.767882:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941460.767889:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941460.767897:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941460.767902:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941460.767905:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941460.767912:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec500. 00010000:00000001:1.0:1773941460.767918:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941460.767922:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138e600. 00000020:00000001:1.0:1773941460.767927:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941460.767930:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138e600 with handle 0xa33a5be14eae1d01 to hash 00000020:00000001:1.0:1773941460.767933:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941460.767934:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941460.767938:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941460.767942:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941460.767959:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941460.767964:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d01 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941460.767977:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d01 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941460.767990:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d01 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941460.767997:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941460.767998:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941460.768005:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941460.768008:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941460.768010:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941460.768012:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121074944 : -128249588476672 : ffff8b5b90d61500) 00000100:00000001:1.0:1773941460.768017:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941460.768022:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941460.768024:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941460.768026:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941460.768036:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d61500 x1860112241152640/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941460.768046:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941460.768048:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941460.768052:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941460.768055:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941460.768058:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941460.768062:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d61500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241152640:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941460.768067:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941460.768071:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941460.768074:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941460.768076:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941460.768081:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0737400. 02000000:00000001:1.0:1773941460.768088:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941460.768096:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941460.768101:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941460.768109:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941460.768116:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241152640, portal 25 00000100:00000001:1.0:1773941460.768118:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941460.768120:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121074944 : -128249588476672 : ffff8b5b90d61500) 00000100:00000040:1.0:1773941460.768126:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d61500 x1860112241152640/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941476 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941460.768134:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941460.768137:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941460.768140:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1908. 00000100:00000200:1.0:1773941460.768144:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241152640, offset 0 00000400:00000200:1.0:1773941460.768150:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941460.768159:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941460.768166:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941460.768172:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941460.768176:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941460.768181:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941460.768183:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941460.768187:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[66, 65] 00000400:00000200:1.0:1773941460.768191:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941460.768195:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [172:172:256:4294967295] 192.168.204.153@tcp LPNI seq info [67:67:8:4294967295] 00000400:00000200:1.0:1773941460.768200:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941460.768210:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941460.768218:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941460.768225:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024bb00. 00000800:00000200:1.0:1773941460.768234:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941460.768243:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941460.768247:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024bb00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941460.768266:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941460.768276:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941460.768279:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294901131 00000100:00000001:1.0:1773941460.768286:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294899085 00000100:00000001:1.0:1773941460.768293:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294901131 00000100:00000001:1.0:1773941460.768298:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294899085 00000100:00000001:1.0:1773941460.768301:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941460.768305:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941460.768307:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941460.768309:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941460.768313:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941460.768315:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941460.768318:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941460.768378:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941460.768382:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024bb00. 00000400:00000200:3.0:1773941460.768387:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941460.768392:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941460.768396:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1908 00000400:00000010:3.0:1773941460.768398:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1908. 00000100:00000001:3.0:1773941460.768401:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941460.768405:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61500 x1860112241152640/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941476 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941460.768413:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941460.768416:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d61500 x1860112241152640/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941476 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941460.768422:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941460.768423:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941460.769412:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941460.769423:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941460.769426:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941460.769429:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941460.769434:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941460.769440:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941460.769447:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941460.769501:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941460.769506:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941460.769522:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605280 00000400:00000200:3.0:1773941460.769527:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941460.769532:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xdf5 [1] + 224 00000800:00000001:3.0:1773941460.769538:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941460.769547:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941460.769549:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941460.769553:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941460.769557:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941460.769562:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61500 x1860112241152640/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941476 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941460.769572:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d61500 x1860112241152640/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941476 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941460.769591:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941460.769598:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941460.769602:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941460.769623:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941460.769629:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941460.769630:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941460.769632:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941460.769635:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941460.769637:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941460.769639:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941460.769643:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61500 x1860112241152640/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941476 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941460.769651:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61500 x1860112241152640/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941476 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941460.769656:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941460.769657:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941460.769659:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941460.769661:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941460.769663:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941460.769664:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941460.769666:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941460.769672:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=39 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941460.769676:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941460.769679:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941460.769684:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941460.769696:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941460.769699:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941460.769723:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941460.769728:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941460.769732:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941460.769736:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941460.769737:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941460.769739:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941460.769740:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941460.769743:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d61500 x1860112241152640/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941476 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941460.769749:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941460.769750:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941460.769751:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941460.769754:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941460.769756:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941460.769757:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941460.769759:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941460.769759:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941460.769760:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941460.769761:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941460.769762:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941460.769764:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941460.769765:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941460.769767:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941460.769769:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 229 00000400:00000200:1.0:1773941460.769773:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941460.769776:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941460.769778:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941460.769781:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941460.769783:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941460.769784:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941460.769791:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941460.769792:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941460.769794:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941460.769796:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941460.769797:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941460.769799:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941460.769801:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941460.769802:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941460.769803:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941460.769805:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941460.769807:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941460.769809:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941460.769811:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941460.769813:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941460.769815:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941460.769817:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941460.769819:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941460.769820:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941460.769823:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61f80. 00000020:00000040:1.0:1773941460.769824:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941460.769827:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941460.769828:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941460.769830:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941460.769832:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941460.769833:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941460.769837:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737000. 00000100:00000001:1.0:1773941460.769840:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941460.769844:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d61f80 x1860112241152768/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941460.769850:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941460.769852:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941460.769855:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002280d596] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d61f80 x1860112241152768/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941460.769873:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941460.769875:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941460.769877:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941460.769879:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941460.769880:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941460.769881:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941460.769883:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941460.769884:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941460.769886:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941460.769889:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d61500 x1860112241152640/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941476 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941460.769895:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941460.769897:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941460.769899:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941460.769901:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941460.769904:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d61500 x1860112241152640/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941476 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941460.769907:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941460.769911:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1773941460.769912:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d61500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241152640:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941460.769914:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941460.769921:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941460.769923:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d61f80 x1860112241152768/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941460.769926:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941460.769933:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941460.769936:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941460.769937:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1773941460.769943:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d61500 x1860112241152640/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941476 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941460.769946:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941460.769951:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941460.769954:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941460.769956:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941460.769957:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941460.769960:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00100000:0.0:1773941460.769964:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241152768:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941460.769965:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941460.769969:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941460.769970:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941460.769972:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:0.0:1773941460.769974:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941460.769976:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941460.769977:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941460.769980:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0c00. 00010000:00000001:1.0:1773941460.769982:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:0.0:1773941460.769983:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941460.769985:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941460.769988:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138e600 refcount=4 00000400:00000200:0.0:1773941460.769989:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:1.0:1773941460.769991:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00000400:00000010:0.0:1773941460.769994:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00000001:1.0:1773941460.769995:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00010000:1.0:1773941460.770002:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d01 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941460.770002:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941460.770007:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241152768, portal 25 00010000:00000001:1.0:1773941460.770013:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941460.770013:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941460.770015:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121077632 : -128249588473984 : ffff8b5b90d61f80) 00010000:00010000:1.0:1773941460.770021:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d01 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941460.770022:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d61f80 x1860112241152768/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941470 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941460.770038:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:0.0:1773941460.770041:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:1.0:1773941460.770043:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d01 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941460.770048:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941460.770053:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c880. 00010000:00000001:1.0:1773941460.770054:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000200:0.0:1773941460.770057:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241152768, offset 0 00010000:00000001:1.0:1773941460.770058:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941460.770061:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d01 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941460.770069:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:0.0:1773941460.770069:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941460.770073:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941460.770076:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:0.0:1773941460.770079:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941460.770080:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d01 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941460.770088:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138e600 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:0.0:1773941460.770089:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 10000000:00000001:1.0:1773941460.770090:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941460.770094:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d01 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941460.770097:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941460.770101:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941460.770104:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941460.770109:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941460.770114:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[66, 67] 00010000:00000001:1.0:1773941460.770126:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:0.0:1773941460.770127:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941460.770129:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941460.770132:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941460.770135:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138e600 refcount=5 00000400:00000200:0.0:1773941460.770136:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [173:173:256:4294967295] 192.168.204.143@tcp LPNI seq info [68:68:8:4294967295] 00000020:00000001:1.0:1773941460.770139:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941460.770142:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941460.770145:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941460.770150:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d01 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941460.770155:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:1.0:1773941460.770165:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941460.770169:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 10000000:00010000:1.0:1773941460.770170:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d01 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941460.770182:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 10000000:01000000:1.0:1773941460.770186:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000800:00000010:0.0:1773941460.770190:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b91e1e300. 10000000:00000040:1.0:1773941460.770191:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941460.770195:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941460.770201:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00010000:1.0:1773941460.770202:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941460.770206:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941460.770209:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941460.770212:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000800:00000200:0.0:1773941460.770213:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941460.770214:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941460.770217:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941460.770221:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138e600 with handle 0xa33a5be14eae1d01 from hash 00000800:00000200:0.0:1773941460.770222:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b91e1e300 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941460.770223:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941460.770225:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941460.770226:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941460.770229:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941460.770230:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941460.770231:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941460.770233:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941460.770234:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941460.770235:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941460.770241:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941460.770242:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941460.770244:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d01 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941460.770252:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941460.770254:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941460.770257:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941460.770258:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294901133 00010000:00000001:1.0:1773941460.770259:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941460.770262:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941460.770264:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941460.770265:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941460.770266:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941460.770267:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294901131 00010000:00010000:1.0:1773941460.770269:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d01 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941460.770276:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294901133 00010000:00000040:1.0:1773941460.770277:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec500 count: 0 00000100:00000001:0.0:1773941460.770286:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294901131 00000100:00000001:0.0:1773941460.770293:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941460.770294:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941460.770295:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941460.770296:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941460.770301:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941460.770301:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941460.770306:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d61500 x1860112241152640/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941476 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941460.770308:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1773941460.770315:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1773941460.770317:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941460.770320:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000800:00000010:3.0:1773941460.770321:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b91e1e300. 02000000:00000001:1.0:1773941460.770321:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1773941460.770325:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000010:1.0:1773941460.770326:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0737400. 00000400:00000200:3.0:1773941460.770328:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:1.0:1773941460.770333:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1773941460.770334:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1773941460.770337:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:3.0:1773941460.770338:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:0.0:1773941460.770339:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000040:1.0:1773941460.770340:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000400:00000200:3.0:1773941460.770342:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c880 00000020:00000001:1.0:1773941460.770344:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941460.770344:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941460.770346:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c880. 02000000:00000010:1.0:1773941460.770349:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 00000100:00000001:0.0:1773941460.770350:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941460.770351:0:2347:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1773941460.770354:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941460.770356:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941460.770358:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61500. 00000100:00000200:3.0:1773941460.770359:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61f80 x1860112241152768/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941470 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941460.770361:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941460.770364:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941460.770366:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941460.770368:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941460.770370:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941460.770376:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0e80. 00000020:00000010:1.0:1773941460.770379:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941460.770382:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941460.770385:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941460.770389:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941460.770392:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941460.770396:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941460.770400:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941460.770407:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 00000100:00000001:3.0:1773941460.770408:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00000040:1.0:1773941460.770411:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000100:00000040:3.0:1773941460.770413:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d61f80 x1860112241152768/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941470 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941460.770415:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941460.770422:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000100:00000001:3.0:1773941460.770427:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:1.0:1773941460.770428:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000100:00000001:3.0:1773941460.770430:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1773941460.770432:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941460.770436:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941460.770439:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941460.770443:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941460.770445:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941460.770447:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941460.770449:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0e80. 10000000:01000000:1.0:1773941460.770452:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941460.770455:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941460.770457:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941460.770458:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941460.770459:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941460.770460:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941460.770462:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941460.771573:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941460.771578:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941460.771580:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941460.771582:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941460.771585:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941460.771588:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941460.771592:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941460.771594:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941460.771596:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941460.771599:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605300 00000400:00000200:3.0:1773941460.771601:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941460.771604:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xdfd [1] + 0 00000800:00000001:3.0:1773941460.771607:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941460.771611:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941460.771658:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941460.771661:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941460.771664:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941460.771667:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61f80 x1860112241152768/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941470 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941460.771674:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d61f80 x1860112241152768/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941470 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941460.771685:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941460.771688:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941460.771691:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941460.771733:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941460.771737:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941460.771741:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941460.771742:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941460.771744:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941460.771749:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941460.771751:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941460.771754:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941460.771760:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61f80 x1860112241152768/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941470 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941460.771782:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61f80 x1860112241152768/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941470 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941460.771789:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941460.771791:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941460.771793:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941460.771795:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941460.771797:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941460.771799:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941460.771801:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941460.771805:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=39 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941460.771808:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941460.771810:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941460.771812:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941460.771814:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941460.771816:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941460.771819:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941460.771823:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d61f80 x1860112241152768/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941470 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941460.771830:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941460.771832:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941460.771834:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941460.771836:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941460.771841:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941460.771843:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941460.771844:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941460.771846:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941460.771848:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941460.771851:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941460.771853:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941460.771857:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941460 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941460.771860:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941460.771866:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941460.771873:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941460.771875:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941460.771877:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941460.771879:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941460.771882:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941460.771884:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941460.771887:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941460.771890:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941460.771894:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941460.771897:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941460.772099:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941460.772102:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941460.772107:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d61f80 x1860112241152768/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941470 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941460.772115:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d61f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241152768:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941460.772119:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941460.772121:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941460.772124:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d61f80 x1860112241152768/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941470 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941460.772126:0:12186:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941460.772132:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941460.772134:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941460.772137:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0c00. 00000100:00080000:0.0:1773941460.772140:0:12186:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 02000000:00000001:2.0:1773941460.772141:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941460.772144:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941460.772147:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941460.772149:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00100000:0.0:1773941460.772149:0:12186:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000010:2.0:1773941460.772152:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737000. 02000000:00000001:2.0:1773941460.772155:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1773941460.772155:0:12186:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941460.772158:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:01000000:0.0:1773941460.772159:0:12186:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000010:2.0:1773941460.772160:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61f80. 00000100:00000001:2.0:1773941460.772166:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:0.0:1773941460.772166:0:12186:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941460.772167:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941460.772171:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:0.0:1773941460.772172:0:12186:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941460.772174:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000001:00000001:0.0:1773941460.772178:0:12186:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941460.772183:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941460.772183:0:12186:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941460.772186:0:12186:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941460.772187:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941460.772190:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941460.772192:0:12186:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941460.772196:0:12186:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941460.772198:0:12186:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941460.772202:0:12186:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941460.772205:0:12186:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941460.772208:0:12186:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941460.772210:0:12186:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941460.772212:0:12186:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941460.772214:0:12186:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941460.772215:0:12186:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941460.772217:0:12186:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941460.772232:0:12186:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941460.772234:0:12186:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941460.772236:0:12186:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941460.772239:0:12186:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941460.772251:0:12186:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941460.772254:0:12186:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941460.772256:0:12186:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941460.772259:0:12186:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941460.772261:0:12186:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941460.776729:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec500. 00010000:00000010:1.1:1773941460.776737:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138e600. 00000020:00000001:0.0:1773941460.831864:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941460.831869:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941460.831872:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941461.855862:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941461.855867:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941461.855871:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941462.175858:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941462.175868:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941462.175874:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941462.175876:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941462.175878:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941462.175880:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941462.175883:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941462.175908:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941462.175911:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941462.175915:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941462.175916:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941462.175925:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61500. 00000020:00000040:1.0:1773941462.175930:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941462.175933:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941462.175936:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941462.175942:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941462.175950:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941462.175958:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941462.175962:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941462.175965:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941462.175973:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec500. 00010000:00000001:1.0:1773941462.175980:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941462.175984:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138e600. 00000020:00000001:1.0:1773941462.175989:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941462.175991:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138e600 with handle 0xa33a5be14eae1d08 to hash 00000020:00000001:1.0:1773941462.175994:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941462.175996:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941462.175999:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941462.176003:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941462.176006:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941462.176011:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d08 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941462.176023:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d08 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941462.176035:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d08 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941462.176041:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941462.176043:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941462.176056:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941462.176059:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941462.176061:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941462.176063:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121074944 : -128249588476672 : ffff8b5b90d61500) 00000100:00000001:1.0:1773941462.176068:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941462.176073:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941462.176075:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941462.176077:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941462.176088:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d61500 x1860112241152896/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941462.176099:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941462.176101:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941462.176105:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941462.176108:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941462.176111:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941462.176115:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d61500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241152896:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941462.176120:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941462.176125:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941462.176128:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941462.176131:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941462.176136:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0737400. 02000000:00000001:1.0:1773941462.176139:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941462.176146:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941462.176151:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941462.176158:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941462.176165:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241152896, portal 25 00000100:00000001:1.0:1773941462.176168:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941462.176169:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121074944 : -128249588476672 : ffff8b5b90d61500) 00000100:00000040:1.0:1773941462.176175:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d61500 x1860112241152896/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941478 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941462.176184:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941462.176187:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941462.176190:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1440. 00000100:00000200:1.0:1773941462.176200:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241152896, offset 0 00000400:00000200:1.0:1773941462.176207:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941462.176216:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941462.176224:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941462.176229:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941462.176234:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941462.176238:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941462.176241:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941462.176244:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[68, 67] 00000400:00000200:1.0:1773941462.176248:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941462.176252:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [174:174:256:4294967295] 192.168.204.153@tcp LPNI seq info [69:69:8:4294967295] 00000400:00000200:1.0:1773941462.176256:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941462.176265:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941462.176269:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941462.176274:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b200. 00000800:00000200:1.0:1773941462.176281:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941462.176286:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941462.176290:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b200 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941462.176305:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941462.176308:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941462.176310:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294902539 00000100:00000001:1.0:1773941462.176316:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294901133 00000100:00000001:1.0:1773941462.176320:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294902539 00000100:00000001:1.0:1773941462.176322:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294901133 00000100:00000001:1.0:1773941462.176325:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941462.176327:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941462.176328:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941462.176330:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941462.176332:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941462.176334:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941462.176341:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941462.176513:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941462.176521:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b200. 00000400:00000200:3.0:1773941462.176532:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941462.176540:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941462.176546:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1440 00000400:00000010:3.0:1773941462.176549:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1440. 00000100:00000001:3.0:1773941462.176565:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941462.176572:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61500 x1860112241152896/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941478 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941462.176582:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941462.176585:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d61500 x1860112241152896/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941478 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941462.176590:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941462.176592:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941462.177591:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.177602:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941462.177604:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.177606:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941462.177612:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941462.177616:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941462.177622:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941462.177625:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941462.177630:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941462.177634:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605380 00000400:00000200:3.0:1773941462.177637:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941462.177721:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xe05 [1] + 224 00000800:00000001:3.0:1773941462.177732:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.177740:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941462.177741:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941462.177745:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941462.177748:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941462.177752:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61500 x1860112241152896/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941478 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941462.177761:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d61500 x1860112241152896/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941478 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941462.177787:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941462.177795:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.177804:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941462.177828:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941462.177837:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941462.177838:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941462.177841:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941462.177845:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941462.177846:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941462.177848:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941462.177853:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61500 x1860112241152896/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941478 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941462.177862:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61500 x1860112241152896/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941478 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941462.177867:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941462.177868:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941462.177870:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941462.177873:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941462.177875:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941462.177877:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941462.177880:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941462.177886:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=41 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941462.177889:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941462.177892:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941462.177897:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941462.177901:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941462.177904:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941462.177907:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941462.177913:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941462.177918:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941462.177921:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941462.177925:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941462.177927:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941462.177929:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941462.177934:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d61500 x1860112241152896/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941478 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941462.177961:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941462.177965:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941462.177968:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941462.177971:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941462.177975:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941462.177977:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941462.177979:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941462.177980:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941462.177981:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941462.177983:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941462.177984:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941462.177987:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941462.177989:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941462.177994:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941462.177997:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 231 00000400:00000200:1.0:1773941462.178003:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941462.178009:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941462.178014:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941462.178018:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941462.178021:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941462.178023:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941462.178028:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941462.178031:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941462.178034:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941462.178037:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941462.178040:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941462.178056:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941462.178061:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941462.178063:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941462.178067:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941462.178070:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941462.178074:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941462.178087:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941462.178090:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941462.178093:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941462.178097:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941462.178099:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941462.178103:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941462.178106:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941462.178112:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61180. 00000020:00000040:1.0:1773941462.178115:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941462.178120:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941462.178122:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941462.178126:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941462.178128:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941462.178133:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941462.178140:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734800. 00000100:00000001:1.0:1773941462.178146:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941462.178154:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d61180 x1860112241153024/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941462.178163:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941462.178167:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941462.178172:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e241f03b] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d61180 x1860112241153024/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941462.178202:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941462.178205:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941462.178208:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941462.178211:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941462.178215:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941462.178217:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941462.178220:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941462.178222:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941462.178224:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941462.178230:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d61500 x1860112241152896/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941478 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941462.178249:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941462.178253:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941462.178256:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941462.178259:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941462.178265:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d61500 x1860112241152896/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941478 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941462.178274:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d61500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241152896:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941462.178279:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941462.178282:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941462.178285:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941462.178287:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941462.178291:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d61500 x1860112241152896/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941478 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941462.178298:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941462.178300:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941462.178305:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941462.178307:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941462.178312:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941462.178316:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941462.178318:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941462.178321:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138e600 refcount=4 00000020:00000001:1.0:1773941462.178323:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941462.178326:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00010000:1.0:1773941462.178333:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d08 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941462.178343:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941462.178349:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d08 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941462.178366:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941462.178370:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d08 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941462.178378:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941462.178380:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941462.178383:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d08 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941462.178391:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941462.178394:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941462.178399:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941462.178403:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d08 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941462.178410:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138e600 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941462.178414:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941462.178416:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d08 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941462.178423:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941462.178425:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941462.178427:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941462.178428:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138e600 refcount=5 00000020:00000001:1.0:1773941462.178431:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941462.178433:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941462.178438:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941462.178441:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d08 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:1.0:1773941462.178448:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941462.178457:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d08 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941462.178465:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941462.178468:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941462.178470:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941462.178475:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941462.178477:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941462.178481:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941462.178494:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941462.178497:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941462.178500:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941462.178504:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138e600 with handle 0xa33a5be14eae1d08 from hash 00010000:00000001:1.0:1773941462.178507:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941462.178509:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941462.178510:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941462.178512:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941462.178513:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941462.178516:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941462.178519:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941462.178520:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941462.178522:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941462.178524:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941462.178528:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d08 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941462.178532:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941462.178533:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941462.178534:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941462.178535:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941462.178537:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941462.178537:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941462.178538:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941462.178539:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d08 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941462.178545:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec500 count: 0 00010000:00000001:1.0:1773941462.178561:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941462.178563:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941462.178565:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941462.178569:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d61500 x1860112241152896/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941478 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941462.178577:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941462.178581:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941462.178584:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0737400. 02000000:00000001:1.0:1773941462.178588:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941462.178590:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941462.178592:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941462.178595:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941462.178599:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941462.178602:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941462.178604:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941462.178606:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61500. 00000100:00000001:1.0:1773941462.178608:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941462.178609:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941462.178611:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941462.178613:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941462.178616:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941462.178619:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0e80. 00000020:00000010:1.0:1773941462.178623:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941462.178628:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941462.178632:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941462.178637:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941462.178642:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941462.178647:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941462.178651:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941462.178656:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941462.178661:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941462.178662:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941462.178668:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941462.178672:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941462.178678:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941462.178685:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941462.178689:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00000100:00000001:2.0:1773941462.178693:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00040000:00000010:1.0:1773941462.178695:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 00000100:00000001:2.0:1773941462.178699:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 20000000:00000010:1.0:1773941462.178702:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000100:00000001:2.0:1773941462.178748:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000020:00000010:1.0:1773941462.178752:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941462.178759:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0e80. 00000100:00000040:2.0:1773941462.178760:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d61180 x1860112241153024/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:01000000:1.0:1773941462.178765:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941462.178773:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941462.178779:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 10000000:00020000:1.0:1773941462.178780:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000100:00000001:2.0:1773941462.178784:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941462.178785:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941462.178789:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 02000000:00000001:2.0:1773941462.178792:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 10000000:00000040:1.0:1773941462.178793:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 02000000:00000001:2.0:1773941462.178797:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 10000000:00000001:1.0:1773941462.178799:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 02000000:00000001:2.0:1773941462.178804:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941462.178809:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241153024:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941462.178814:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941462.178819:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941462.178821:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941462.178824:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941462.178830:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce5800. 02000000:00000001:2.0:1773941462.178833:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941462.178839:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941462.178843:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941462.178850:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000200:2.0:1773941462.178857:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241153024, portal 25 00000100:00000001:2.0:1773941462.178860:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941462.178861:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121074048 : -128249588477568 : ffff8b5b90d61180) 00000100:00000040:2.0:1773941462.178868:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d61180 x1860112241153024/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941472 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941462.178878:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941462.178881:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941462.178884:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34c38. 00000100:00000200:2.0:1773941462.178887:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241153024, offset 0 00000400:00000200:2.0:1773941462.178901:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941462.178907:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941462.178921:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941462.178926:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941462.178930:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941462.178934:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941462.178937:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941462.178940:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[68, 69] 00000400:00000200:2.0:1773941462.178944:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941462.178949:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [175:175:256:4294967295] 192.168.204.143@tcp LPNI seq info [70:70:8:4294967295] 00000400:00000200:2.0:1773941462.178955:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941462.178965:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941462.178972:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941462.178978:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0200. 00000800:00000200:2.0:1773941462.178985:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941462.178993:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941462.178997:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0200 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941462.179014:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941462.179019:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941462.179022:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294902542 00000100:00000001:2.0:1773941462.179029:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294902539 00000100:00000001:2.0:1773941462.179034:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294902542 00000100:00000001:2.0:1773941462.179041:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294902539 00000100:00000001:2.0:1773941462.179045:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941462.179048:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.179050:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.179053:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941462.179057:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941462.179058:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941462.179061:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941462.179062:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941462.179064:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.179066:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941462.179112:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941462.179119:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0200. 00000400:00000200:3.0:1773941462.179125:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941462.179130:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941462.179135:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34c38 00000400:00000010:3.0:1773941462.179137:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34c38. 00000100:00000001:3.0:1773941462.179141:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941462.179146:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61180 x1860112241153024/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941472 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941462.179155:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941462.179159:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d61180 x1860112241153024/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941472 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941462.179167:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941462.179169:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941462.180236:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.180243:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941462.180245:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.180246:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941462.180251:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941462.180255:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941462.180259:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941462.180262:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941462.180265:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941462.180268:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605400 00000400:00000200:3.0:1773941462.180303:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941462.180307:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xe0d [1] + 0 00000800:00000001:3.0:1773941462.180312:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.180317:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941462.180319:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941462.180322:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941462.180325:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941462.180329:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61180 x1860112241153024/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941472 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941462.180342:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d61180 x1860112241153024/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941472 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941462.180355:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941462.180360:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.180362:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941462.180383:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941462.180385:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941462.180389:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941462.180391:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941462.180393:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941462.180396:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941462.180397:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941462.180399:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941462.180404:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61180 x1860112241153024/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941472 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941462.180415:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61180 x1860112241153024/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941472 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941462.180423:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941462.180425:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941462.180427:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941462.180431:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941462.180433:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941462.180435:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941462.180439:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941462.180444:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=41 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941462.180448:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941462.180451:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941462.180453:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941462.180456:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941462.180459:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.180462:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941462.180467:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d61180 x1860112241153024/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941472 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941462.180475:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941462.180477:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941462.180479:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941462.180496:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941462.180502:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941462.180505:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941462.180507:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941462.180519:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941462.180522:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941462.180524:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941462.180527:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941462.180532:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941462 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941462.180535:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941462.180543:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941462.180546:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941462.180549:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941462.180551:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.180553:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941462.180557:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941462.180560:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941462.180563:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941462.180567:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941462.180571:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941462.180574:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941462.180827:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.180832:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941462.180840:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d61180 x1860112241153024/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941472 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941462.180849:0:12187:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941462.180853:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d61180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241153024:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941462.180861:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941462.180863:0:12187:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:2.0:1773941462.180865:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1773941462.180869:0:12187:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:2.0:1773941462.180873:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d61180 x1860112241153024/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941472 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941462.180874:0:12187:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941462.180878:0:12187:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941462.180884:0:12187:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941462.180889:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:1.0:1773941462.180889:0:12187:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941462.180895:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941462.180896:0:12187:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941462.180898:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce5800. 00000001:00000001:1.0:1773941462.180901:0:12187:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941462.180904:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941462.180904:0:12187:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941462.180907:0:12187:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941462.180910:0:12187:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941462.180915:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941462.180915:0:12187:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:2.0:1773941462.180919:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00080000:1.0:1773941462.180922:0:12187:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000020:00000001:2.0:1773941462.180924:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941462.180929:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734800. 00000100:00000001:1.0:1773941462.180930:0:12187:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941462.180933:0:12187:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:2.0:1773941462.180936:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941462.180936:0:12187:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941462.180941:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:1.0:1773941462.180941:0:12187:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000010:2.0:1773941462.180946:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61180. 00000100:00000001:1.0:1773941462.180946:0:12187:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941462.180949:0:12187:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941462.180951:0:12187:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:2.0:1773941462.180953:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941462.180956:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941462.180959:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941462.180963:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941462.180965:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.180968:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1773941462.180970:0:12187:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:2.0:1773941462.180971:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941462.180978:0:12187:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941462.180981:0:12187:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941462.180985:0:12187:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941462.180989:0:12187:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941462.180991:0:12187:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941462.180993:0:12187:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941462.180995:0:12187:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941462.180996:0:12187:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941462.183753:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec500. 00010000:00000010:1.1:1773941462.183764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138e600. 00000020:00000001:0.0:1773941462.879923:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941462.879933:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941462.879940:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1773941462.943897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1773941462.943912:0:36:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1773941462.943928:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90274700. 00000020:00000040:3.0:1773941462.943936:0:36:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941462.943943:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941462.943947:0:36:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941462.943955:0:36:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941462.943974:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b98a69700. 00000100:00000001:3.0:1773941462.943983:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941462.943998:0:36:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS req@ffff8b5b90274700 x1860112241153152/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941462.944013:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941462.944019:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941462.944036:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2e4aa82] to pc [ptlrpcd_01_01+1] req@ffff8b5b90274700 x1860112241153152/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941462.944073:0:36:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941462.944079:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (238) 00000100:00000001:2.0:1773941462.944147:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941462.944153:0:2355:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941462.944156:0:2355:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941462.944163:0:2355:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90274700 x1860112241153152/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941462.944170:0:2355:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941462.944172:0:2355:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941462.944176:0:2355:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941462.944179:0:2355:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941462.944181:0:2355:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941462.944184:0:2355:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90274700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2355:1860112241153152:192.168.204.143@tcp:400:kworker.0 00000100:00000001:2.0:1773941462.944188:0:2355:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941462.944194:0:2355:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941462.944196:0:2355:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941462.944198:0:2355:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941462.944203:0:2355:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b90e7f400. 02000000:00000001:2.0:1773941462.944205:0:2355:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941462.944213:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941462.944217:0:2355:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941462.944225:0:2355:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000200:2.0:1773941462.944233:0:2355:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241153152, portal 25 00000100:00000001:2.0:1773941462.944235:0:2355:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941462.944254:0:2355:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109618944 : -128249599932672 : ffff8b5b90274700) 00000100:00000040:2.0:1773941462.944259:0:2355:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90274700 x1860112241153152/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941462.944266:0:2355:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941462.944268:0:2355:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941462.944270:0:2355:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34440. 00000100:00000200:2.0:1773941462.944273:0:2355:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241153152, offset 0 00000400:00000200:2.0:1773941462.944280:0:2355:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941462.944289:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941462.944298:0:2355:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941462.944303:0:2355:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941462.944307:0:2355:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941462.944311:0:2355:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941462.944313:0:2355:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941462.944316:0:2355:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[70, 69] 00000400:00000200:2.0:1773941462.944319:0:2355:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:2.0:1773941462.944323:0:2355:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [176:176:256:4294967295] 192.168.204.153@tcp LPNI seq info [71:71:8:4294967295] 00000400:00000200:2.0:1773941462.944326:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941462.944334:0:2355:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:2.0:1773941462.944339:0:2355:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:2.0:1773941462.944344:0:2355:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0c00. 00000800:00000200:2.0:1773941462.944350:0:2355:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:2.0:1773941462.944358:0:2355:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:2.0:1773941462.944363:0:2355:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0c00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1773941462.944377:0:2355:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941462.944381:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941462.944384:0:2355:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294903307 00000100:00000001:2.0:1773941462.944390:0:2355:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294902542 00000100:00000001:2.0:1773941462.944393:0:2355:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294903307 00000100:00000001:2.0:1773941462.944397:0:2355:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294902542 00000100:00000001:2.0:1773941462.944404:0:2355:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941462.944406:0:2355:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.944410:0:2355:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.944412:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941462.944419:0:2355:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941462.944421:0:2355:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1773941462.944423:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941462.944424:0:2355:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941462.944425:0:2355:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.944427:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.944438:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941462.944439:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941462.944525:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941462.944530:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0c00. 00000400:00000200:3.0:1773941462.944536:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941462.944542:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941462.944547:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34440 00000400:00000010:3.0:1773941462.944550:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34440. 00000100:00000001:3.0:1773941462.944554:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941462.944558:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90274700 x1860112241153152/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941462.944576:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941462.944579:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90274700 x1860112241153152/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941478 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941462.944585:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941462.944586:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941462.945590:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.945603:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941462.945606:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.945609:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941462.945615:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941462.945621:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941462.945629:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941462.945633:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941462.945638:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941462.945643:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605480 00000400:00000200:3.0:1773941462.945693:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941462.945700:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xe1d [1] + 224 00000800:00000001:3.0:1773941462.945742:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.945751:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941462.945754:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941462.945759:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941462.945765:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941462.945771:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90274700 x1860112241153152/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941478 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941462.945782:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b90274700 x1860112241153152/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941478 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941462.945800:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941462.945808:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.945812:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941462.945836:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941462.945838:0:2355:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941462.945842:0:2355:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941462.945844:0:2355:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941462.945846:0:2355:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941462.945849:0:2355:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941462.945850:0:2355:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941462.945852:0:2355:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941462.945856:0:2355:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90274700 x1860112241153152/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941478 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941462.945864:0:2355:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90274700 x1860112241153152/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941478 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941462.945873:0:2355:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941462.945875:0:2355:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941462.945877:0:2355:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941462.945880:0:2355:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941462.945882:0:2355:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941462.945885:0:2355:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941462.945887:0:2355:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941462.945893:0:2355:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=41 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941462.945895:0:2355:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941462.945898:0:2355:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941462.945900:0:2355:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941462.945903:0:2355:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941462.945910:0:2355:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941462.945914:0:2355:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941462.945918:0:2355:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941462.945921:0:2355:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941462.945923:0:2355:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941462.945925:0:2355:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941462.945926:0:2355:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941462.945930:0:2355:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90274700 x1860112241153152/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941478 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941462.945935:0:2355:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941462.945935:0:2355:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941462.945937:0:2355:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941462.945941:0:2355:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941462.945943:0:2355:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941462.945944:0:2355:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.945945:0:2355:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941462.945946:0:2355:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941462.945947:0:2355:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941462.945948:0:2355:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.945949:0:2355:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941462.945951:0:2355:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941462.945953:0:2355:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941462.945955:0:2355:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941462.945958:0:2355:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 232 00000400:00000200:2.0:1773941462.945961:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941462.945964:0:2355:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941462.945968:0:2355:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941462.945970:0:2355:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941462.945972:0:2355:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941462.945974:0:2355:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:2.0:1773941462.945977:0:2355:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941462.945979:0:2355:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941462.945982:0:2355:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941462.945989:0:2355:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941462.945992:0:2355:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941462.945994:0:2355:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:2.0:1773941462.945998:0:2355:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941462.946000:0:2355:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941462.946002:0:2355:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941462.946004:0:2355:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:2.0:1773941462.946007:0:2355:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:2.0:1773941462.946011:0:2355:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:2.0:1773941462.946014:0:2355:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941462.946017:0:2355:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941462.946021:0:2355:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941462.946024:0:2355:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.946027:0:2355:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941462.946030:0:2355:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941462.946034:0:2355:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021d500. 00000020:00000040:2.0:1773941462.946038:0:2355:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:2.0:1773941462.946041:0:2355:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941462.946043:0:2355:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.946046:0:2355:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941462.946048:0:2355:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941462.946051:0:2355:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941462.946063:0:2355:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0ce5800. 00000100:00000001:2.0:1773941462.946068:0:2355:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941462.946075:0:2355:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b9021d500 x1860112241153280/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941462.946084:0:2355:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941462.946086:0:2355:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941462.946090:0:2355:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1896fa] to pc [ptlrpcd_rcv+-1] req@ffff8b5b9021d500 x1860112241153280/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941462.946111:0:2355:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.946114:0:2355:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941462.946121:0:2355:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:2.0:1773941462.946128:0:2355:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90274700 x1860112241153152/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941478 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941462.946137:0:2355:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941462.946140:0:2355:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941462.946143:0:2355:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941462.946147:0:2355:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90274700 x1860112241153152/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941478 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941462.946157:0:2355:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90274700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2355:1860112241153152:192.168.204.143@tcp:400:kworker.0 00000100:00000001:2.0:1773941462.946163:0:2355:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941462.946166:0:2355:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941462.946171:0:2355:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90274700 x1860112241153152/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941478 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941462.946174:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941462.946178:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941462.946181:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1773941462.946183:0:2355:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941462.946187:0:2355:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:0.0:1773941462.946189:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021d500 x1860112241153280/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1773941462.946192:0:2355:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b90e7f400. 02000000:00000001:2.0:1773941462.946200:0:2355:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1773941462.946202:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1773941462.946204:0:2355:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941462.946205:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941462.946209:0:2355:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 02000000:00000001:0.0:1773941462.946209:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941462.946212:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1773941462.946215:0:2355:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:0.0:1773941462.946216:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941462.946221:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241153280:192.168.204.143@tcp:250:ptlrpcd_01_01.0 02000000:00000010:2.0:1773941462.946222:0:2355:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b98a69700. 00000100:00000001:0.0:1773941462.946225:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941462.946228:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941462.946229:0:2355:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941462.946230:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941462.946233:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1773941462.946234:0:2355:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 02000000:00000010:0.0:1773941462.946238:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c3000. 00000100:00000010:2.0:1773941462.946239:0:2355:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90274700. 02000000:00000001:0.0:1773941462.946242:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.946245:0:2355:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000400:00000200:0.0:1773941462.946248:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941462.946250:0:2355:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1773941462.946253:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000100:00000001:2.0:1773941462.946255:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1773941462.946258:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000001:2.0:1773941462.946262:0:2355:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000200:0.0:1773941462.946262:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241153280, portal 25 00000100:00000001:2.0:1773941462.946266:0:2355:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941462.946267:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941462.946269:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109262080 : -128249600289536 : ffff8b5b9021d500) 00000100:00000001:2.0:1773941462.946272:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941462.946274:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941462.946282:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021d500 x1860112241153280/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941472 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941462.946289:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941462.946291:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941462.946293:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6cc38. 00000100:00000200:0.0:1773941462.946297:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241153280, offset 0 00000400:00000200:0.0:1773941462.946301:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941462.946304:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941462.946308:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941462.946310:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941462.946313:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941462.946315:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941462.946317:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941462.946319:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[70, 71] 00000400:00000200:0.0:1773941462.946321:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941462.946323:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [177:177:256:4294967295] 192.168.204.143@tcp LPNI seq info [72:72:8:4294967295] 00000400:00000200:0.0:1773941462.946326:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941462.946332:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941462.946335:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941462.946340:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b91e1e300. 00000800:00000200:0.0:1773941462.946343:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941462.946349:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941462.946352:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b91e1e300 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:0.0:1773941462.946363:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941462.946366:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941462.946368:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294903309 00000100:00000001:0.0:1773941462.946371:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294903307 00000100:00000001:0.0:1773941462.946374:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294903309 00000100:00000001:0.0:1773941462.946376:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294903307 00000100:00000001:0.0:1773941462.946378:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941462.946384:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941462.946387:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941462.946390:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941462.946395:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941462.946397:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1773941462.946400:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941462.946402:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941462.946403:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941462.946406:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941462.946454:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941462.946458:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b91e1e300. 00000400:00000200:3.0:1773941462.946483:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941462.946488:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941462.946493:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6cc38 00000400:00000010:3.0:1773941462.946495:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6cc38. 00000100:00000001:3.0:1773941462.946499:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941462.946504:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021d500 x1860112241153280/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941472 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941462.946513:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941462.946517:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021d500 x1860112241153280/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941472 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941462.946578:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941462.946580:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941462.947629:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.947636:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941462.947639:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.947641:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941462.947645:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941462.947694:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941462.947701:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941462.947723:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941462.947727:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941462.947730:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605500 00000400:00000200:3.0:1773941462.947734:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941462.947738:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xe2d [1] + 0 00000800:00000001:3.0:1773941462.947743:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.947751:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941462.947754:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941462.947757:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941462.947761:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941462.947766:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021d500 x1860112241153280/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941472 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941462.947777:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b9021d500 x1860112241153280/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941472 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941462.947800:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941462.947806:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941462.947809:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941462.947862:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941462.947864:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941462.947868:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941462.947869:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941462.947871:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941462.947874:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941462.947876:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941462.947878:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941462.947883:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021d500 x1860112241153280/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941472 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941462.947890:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021d500 x1860112241153280/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941472 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941462.947896:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941462.947897:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941462.947898:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941462.947899:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941462.947900:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941462.947901:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941462.947903:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941462.947905:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=41 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941462.947908:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941462.947909:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941462.947911:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941462.947913:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941462.947915:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941462.947917:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941462.947920:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021d500 x1860112241153280/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941472 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941462.947935:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941462.947937:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941462.947938:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941462.947940:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941462.947946:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941462.947948:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941462.947949:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941462.947950:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941462.947952:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941462.947953:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941462.947955:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941462.947958:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941462 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941462.947960:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941462.947966:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941462.947967:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941462.947969:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941462.947970:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941462.947971:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941462.947974:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941462.947976:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941462.947977:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941462.947981:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941462.947983:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941462.947985:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941462.948256:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.948267:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941462.948274:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021d500 x1860112241153280/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941472 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941462.948278:0:12188:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941462.948286:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241153280:192.168.204.143@tcp:250:ptlrpcd_01_01.0 00000100:00080000:0.0:1773941462.948289:0:12188:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:2.0:1773941462.948294:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941462.948295:0:12188:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941462.948297:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941462.948299:0:12188:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941462.948301:0:12188:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000040:2.0:1773941462.948303:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021d500 x1860112241153280/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941472 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000001:00000001:0.0:1773941462.948305:0:12188:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941462.948310:0:12188:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941462.948311:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941462.948313:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941462.948315:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c3000. 02000000:00000001:2.0:1773941462.948319:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941462.948321:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941462.948321:0:12188:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941462.948322:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941462.948327:0:12188:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941462.948328:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:0.0:1773941462.948331:0:12188:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941462.948332:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0ce5800. 00000001:00000001:0.0:1773941462.948334:0:12188:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941462.948336:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000001:0.0:1773941462.948337:0:12188:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941462.948338:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941462.948340:0:12188:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000010:2.0:1773941462.948341:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021d500. 00000100:00000001:2.0:1773941462.948345:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:0.0:1773941462.948346:0:12188:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941462.948347:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941462.948352:0:12188:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941462.948353:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941462.948357:0:12188:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941462.948359:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941462.948360:0:12188:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941462.948361:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941462.948364:0:12188:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941462.948365:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941462.948368:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941462.948369:0:12188:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941462.948372:0:12188:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941462.948373:0:12188:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941462.948393:0:12188:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941462.948395:0:12188:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941462.948396:0:12188:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941462.948401:0:12188:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941462.948405:0:12188:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941462.948408:0:12188:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941462.948410:0:12188:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941462.948413:0:12188:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941462.948415:0:12188:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941463.903885:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941463.903894:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941463.903898:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941464.095864:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941464.095878:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941464.095883:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941464.095885:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941464.095887:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941464.095888:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941464.095891:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941464.095893:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941464.095896:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941464.095898:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941464.095900:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941464.095913:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61500. 00000020:00000040:1.0:1773941464.095919:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941464.095924:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941464.095928:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941464.095934:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941464.095941:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941464.095949:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941464.095955:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941464.095959:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941464.095969:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec500. 00010000:00000001:1.0:1773941464.095977:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941464.095981:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138e600. 00000020:00000001:1.0:1773941464.095986:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941464.095990:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138e600 with handle 0xa33a5be14eae1d0f to hash 00000020:00000001:1.0:1773941464.095993:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941464.095994:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941464.095998:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941464.096035:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941464.096038:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941464.096043:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d0f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941464.096053:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d0f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941464.096065:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d0f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941464.096070:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941464.096072:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941464.096079:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941464.096080:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941464.096083:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941464.096084:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121074944 : -128249588476672 : ffff8b5b90d61500) 00000100:00000001:1.0:1773941464.096100:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941464.096104:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941464.096106:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941464.096108:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941464.096119:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d61500 x1860112241153408/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941464.096127:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941464.096128:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941464.096132:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941464.096135:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941464.096136:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941464.096139:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d61500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241153408:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941464.096143:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941464.096149:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941464.096151:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941464.096153:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941464.096162:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0737400. 02000000:00000001:1.0:1773941464.096163:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941464.096171:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941464.096175:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941464.096181:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941464.096188:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241153408, portal 25 00000100:00000001:1.0:1773941464.096190:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941464.096191:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121074944 : -128249588476672 : ffff8b5b90d61500) 00000100:00000040:1.0:1773941464.096195:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d61500 x1860112241153408/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941480 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941464.096201:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941464.096203:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941464.096205:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1c38. 00000100:00000200:1.0:1773941464.096208:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241153408, offset 0 00000400:00000200:1.0:1773941464.096213:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941464.096222:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941464.096230:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941464.096236:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941464.096239:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941464.096243:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941464.096246:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941464.096248:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[72, 71] 00000400:00000200:1.0:1773941464.096251:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941464.096254:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [178:178:256:4294967295] 192.168.204.153@tcp LPNI seq info [73:73:8:4294967295] 00000400:00000200:1.0:1773941464.096258:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941464.096267:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941464.096272:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941464.096276:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b900. 00000800:00000200:1.0:1773941464.096283:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941464.096291:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941464.096296:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b900 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941464.096317:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941464.096321:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941464.096324:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294904459 00000100:00000001:1.0:1773941464.096331:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294903309 00000100:00000001:1.0:1773941464.096337:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294904459 00000100:00000001:1.0:1773941464.096341:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294903309 00000100:00000001:1.0:1773941464.096344:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941464.096346:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941464.096347:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941464.096349:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941464.096352:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941464.096356:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941464.096361:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941464.096475:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941464.096483:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b900. 00000400:00000200:3.0:1773941464.096490:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941464.096498:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941464.096504:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1c38 00000400:00000010:3.0:1773941464.096506:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1c38. 00000100:00000001:3.0:1773941464.096509:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941464.096516:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61500 x1860112241153408/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941480 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941464.096526:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941464.096529:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d61500 x1860112241153408/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941480 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941464.096534:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941464.096536:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941464.097552:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941464.097561:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941464.097563:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941464.097565:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941464.097570:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941464.097574:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941464.097580:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941464.097583:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941464.097588:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941464.097592:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605580 00000400:00000200:3.0:1773941464.097595:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941464.097600:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xe35 [1] + 224 00000800:00000001:3.0:1773941464.097672:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941464.097687:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941464.097689:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941464.097693:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941464.097696:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941464.097699:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61500 x1860112241153408/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941480 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941464.097734:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d61500 x1860112241153408/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941480 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941464.097748:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941464.097754:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941464.097757:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941464.097787:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941464.097794:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941464.097796:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941464.097799:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941464.097804:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941464.097806:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941464.097809:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941464.097817:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61500 x1860112241153408/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941480 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941464.097829:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61500 x1860112241153408/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941480 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941464.097837:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941464.097840:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941464.097843:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941464.097846:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941464.097849:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941464.097852:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941464.097865:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941464.097876:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=43 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941464.097879:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941464.097881:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941464.097893:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941464.097895:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941464.097898:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941464.097900:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941464.097904:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941464.097908:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941464.097911:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941464.097913:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941464.097914:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941464.097916:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941464.097919:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d61500 x1860112241153408/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941480 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941464.097924:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941464.097925:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941464.097927:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941464.097929:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941464.097931:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941464.097933:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941464.097934:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941464.097935:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941464.097936:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941464.097937:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941464.097938:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941464.097941:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941464.097942:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941464.097944:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941464.097947:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 233 00000400:00000200:1.0:1773941464.097952:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941464.097957:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941464.097962:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941464.097965:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941464.097967:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941464.097975:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941464.097979:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941464.097982:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941464.097984:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941464.097987:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941464.097990:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941464.097992:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941464.097996:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941464.097998:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941464.098001:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941464.098004:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941464.098007:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941464.098011:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941464.098015:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941464.098018:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941464.098021:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941464.098023:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941464.098025:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941464.098027:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941464.098031:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60e00. 00000020:00000040:1.0:1773941464.098033:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941464.098035:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941464.098037:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941464.098039:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941464.098040:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941464.098042:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941464.098046:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0735400. 00000100:00000001:1.0:1773941464.098049:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941464.098054:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60e00 x1860112241153536/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941464.098060:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941464.098062:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941464.098070:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5e4d003] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60e00 x1860112241153536/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941464.098089:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941464.098091:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941464.098092:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941464.098094:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941464.098096:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941464.098097:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941464.098099:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941464.098101:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941464.098102:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941464.098106:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d61500 x1860112241153408/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941480 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941464.098112:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941464.098115:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941464.098117:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941464.098119:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941464.098123:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d61500 x1860112241153408/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941480 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941464.098126:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1773941464.098129:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d61500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241153408:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941464.098132:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941464.098134:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941464.098135:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941464.098138:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941464.098140:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941464.098143:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941464.098145:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60e00 x1860112241153536/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941464.098149:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d61500 x1860112241153408/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941480 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941464.098167:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941464.098171:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941464.098172:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941464.098178:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:1.0:1773941464.098178:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941464.098183:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941464.098184:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941464.098189:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941464.098192:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941464.098197:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00100000:2.0:1773941464.098199:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241153536:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941464.098202:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:2.0:1773941464.098203:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:1.0:1773941464.098205:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:2.0:1773941464.098206:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:1.0:1773941464.098208:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138e600 refcount=4 02000000:00000001:2.0:1773941464.098209:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941464.098211:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 02000000:00000001:2.0:1773941464.098212:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941464.098215:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 02000000:00000010:2.0:1773941464.098218:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce5800. 00010000:00010000:1.0:1773941464.098219:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d0f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941464.098223:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941464.098231:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941464.098231:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000010:2.0:1773941464.098235:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00010000:00010000:1.0:1773941464.098237:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d0f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941464.098241:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00000001:1.0:1773941464.098243:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000200:2.0:1773941464.098246:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241153536, portal 25 00010000:00010000:1.0:1773941464.098247:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d0f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941464.098251:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941464.098254:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00010000:00000001:1.0:1773941464.098257:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000040:2.0:1773941464.098261:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60e00 x1860112241153536/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941474 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941464.098262:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941464.098265:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d0f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941464.098284:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:2.0:1773941464.098285:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941464.098287:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941464.098291:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941464.098293:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000010:2.0:1773941464.098294:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34908. 00000100:00000200:2.0:1773941464.098298:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241153536, offset 0 00010000:00010000:1.0:1773941464.098298:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d0f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941464.098307:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941464.098309:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138e600 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:2.0:1773941464.098313:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:1.0:1773941464.098313:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941464.098317:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d0f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941464.098320:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941464.098327:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941464.098330:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941464.098336:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941464.098341:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941464.098344:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:2.0:1773941464.098345:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[72, 73] 00010000:00000001:1.0:1773941464.098348:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941464.098352:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000020:00000001:1.0:1773941464.098352:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941464.098356:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138e600 refcount=5 00000400:00000200:2.0:1773941464.098357:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [179:179:256:4294967295] 192.168.204.143@tcp LPNI seq info [74:74:8:4294967295] 00000020:00000001:1.0:1773941464.098358:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00010000:00000001:1.0:1773941464.098363:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859115520 : -128249850436096 : ffff8b5b8138e600) 00000400:00000200:2.0:1773941464.098365:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941464.098369:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941464.098371:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d0f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:1.0:1773941464.098379:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941464.098383:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d0f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941464.098386:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 10000000:01000000:1.0:1773941464.098397:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000800:00000200:2.0:1773941464.098398:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 10000000:00000040:1.0:1773941464.098402:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000800:00000010:2.0:1773941464.098404:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0a00. 10000000:00000001:1.0:1773941464.098405:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941464.098411:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941464.098413:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000800:00000200:2.0:1773941464.098414:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941464.098416:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941464.098419:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941464.098420:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941464.098422:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941464.098424:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000020:00000040:1.0:1773941464.098426:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138e600 with handle 0xa33a5be14eae1d0f from hash 00000800:00000200:2.0:1773941464.098429:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0a00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941464.098430:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941464.098433:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941464.098435:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941464.098436:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941464.098437:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941464.098439:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941464.098440:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941464.098441:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941464.098442:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941464.098458:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941464.098464:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941464.098466:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941464.098469:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d0f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941464.098470:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294904461 00000100:00000001:2.0:1773941464.098476:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294904459 00010000:00000001:1.0:1773941464.098480:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941464.098482:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294904461 00010000:00000001:1.0:1773941464.098483:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941464.098487:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941464.098490:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941464.098492:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941464.098493:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294904459 00010000:00000001:1.0:1773941464.098496:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941464.098498:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941464.098499:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941464.098503:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138e600/0xa33a5be14eae1d0f lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941464.098504:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941464.098511:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1773941464.098513:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec500 count: 0 00000100:00000001:2.0:1773941464.098515:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941464.098522:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941464.098524:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941464.098527:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941464.098529:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941464.098531:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941464.098534:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941464.098535:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941464.098538:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941464.098539:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941464.098545:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d61500 x1860112241153408/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941480 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941464.098551:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941464.098554:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941464.098556:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0737400. 02000000:00000001:1.0:1773941464.098560:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000800:00000200:3.0:1773941464.098562:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000020:00000001:1.0:1773941464.098563:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941464.098568:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000800:00000010:3.0:1773941464.098569:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0a00. 00000400:00000200:3.0:1773941464.098578:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:1.0:1773941464.098578:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941464.098583:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 00000400:00000200:3.0:1773941464.098587:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941464.098593:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34908 02000000:00000001:1.0:1773941464.098593:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000400:00000010:3.0:1773941464.098601:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34908. 02000000:00000001:1.0:1773941464.098601:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941464.098609:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61500. 00000100:00000001:1.0:1773941464.098615:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941464.098617:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941464.098620:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941464.098623:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941464.098626:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941464.098635:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0e80. 00000020:00000010:1.0:1773941464.098639:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941464.098644:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000100:00000001:3.0:1773941464.098646:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000040:00000010:1.0:1773941464.098647:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941464.098654:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000100:00000200:3.0:1773941464.098655:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60e00 x1860112241153536/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941474 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941464.098665:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 00000100:00000001:3.0:1773941464.098671:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 80000000:00000010:1.0:1773941464.098673:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000040:3.0:1773941464.098679:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60e00 x1860112241153536/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941474 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00040000:00000010:1.0:1773941464.098681:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 00000100:00000001:3.0:1773941464.098694:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 20000000:00000010:1.0:1773941464.098694:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 00000100:00000001:3.0:1773941464.098724:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000040:1.0:1773941464.098746:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941464.098750:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941464.098756:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941464.098762:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941464.098768:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941464.098775:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941464.098778:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941464.098784:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941464.098788:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941464.098790:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941464.098792:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0e80. 10000000:01000000:1.0:1773941464.098795:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941464.098797:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941464.098799:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941464.098800:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941464.098801:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941464.098802:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941464.098804:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941464.099763:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941464.099769:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941464.099772:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941464.099774:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941464.099780:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941464.099785:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941464.099791:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941464.099846:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941464.099849:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941464.099852:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605600 00000400:00000200:3.0:1773941464.099855:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941464.099858:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xe3d [1] + 0 00000800:00000001:3.0:1773941464.099862:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941464.099867:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941464.099869:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941464.099871:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941464.099874:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941464.099878:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60e00 x1860112241153536/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941474 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941464.099886:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60e00 x1860112241153536/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941474 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941464.099900:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941464.099904:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941464.099906:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941464.099932:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941464.099936:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941464.099939:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941464.099940:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941464.099943:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941464.099947:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941464.099949:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941464.099951:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941464.099957:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60e00 x1860112241153536/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941474 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941464.099965:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60e00 x1860112241153536/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941474 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941464.099977:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941464.099978:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941464.099980:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941464.099982:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941464.099983:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941464.099985:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941464.099986:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941464.099989:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=43 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941464.099991:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941464.099993:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941464.099994:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941464.099996:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941464.099998:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941464.100000:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941464.100004:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60e00 x1860112241153536/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941474 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941464.100008:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941464.100010:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941464.100011:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941464.100013:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941464.100017:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941464.100019:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941464.100020:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941464.100022:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941464.100023:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941464.100025:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941464.100027:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941464.100030:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941464 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941464.100032:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941464.100037:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941464.100039:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941464.100041:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941464.100042:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941464.100047:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941464.100050:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941464.100052:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941464.100054:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941464.100056:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941464.100059:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941464.100061:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941464.100265:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941464.100268:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941464.100272:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60e00 x1860112241153536/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941474 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941464.100279:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241153536:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941464.100283:0:12189:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941464.100284:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941464.100288:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941464.100291:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60e00 x1860112241153536/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941474 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941464.100296:0:12189:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:2.0:1773941464.100300:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941464.100303:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:1.0:1773941464.100303:0:12189:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000010:2.0:1773941464.100307:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce5800. 00000100:00000001:1.0:1773941464.100307:0:12189:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941464.100311:0:12189:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941464.100312:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941464.100315:0:12189:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941464.100316:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941464.100319:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:1.0:1773941464.100320:0:12189:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:2.0:1773941464.100321:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941464.100323:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0735400. 02000000:00000001:2.0:1773941464.100326:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941464.100327:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941464.100330:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60e00. 00000001:00000001:1.0:1773941464.100332:0:12189:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941464.100333:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:1.0:1773941464.100336:0:12189:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941464.100337:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941464.100339:0:12189:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941464.100340:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941464.100342:0:12189:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941464.100345:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000001:1.0:1773941464.100345:0:12189:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941464.100348:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941464.100349:0:12189:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941464.100353:0:12189:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941464.100357:0:12189:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941464.100359:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941464.100360:0:12189:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941464.100362:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941464.100363:0:12189:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941464.100367:0:12189:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941464.100369:0:12189:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941464.100370:0:12189:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941464.100371:0:12189:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941464.100390:0:12189:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941464.100391:0:12189:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941464.100392:0:12189:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941464.100395:0:12189:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941464.100397:0:12189:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941464.100399:0:12189:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941464.100401:0:12189:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941464.100403:0:12189:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941464.100404:0:12189:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941464.103743:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec500. 00010000:00000010:1.1:1773941464.103750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138e600. 00000020:00000001:0.0:1773941464.927884:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941464.927894:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941464.927902:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941465.951918:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941465.951926:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941465.951931:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941466.015799:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941466.015811:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941466.015819:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941466.015823:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941466.015825:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941466.015827:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941466.015831:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941466.015834:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941466.015838:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941466.015840:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941466.015843:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941466.015858:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277b80. 00000020:00000040:3.0:1773941466.015866:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941466.015872:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941466.015901:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941466.015909:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941466.015920:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e3000. 00000100:00000001:3.0:1773941466.015930:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941466.015936:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941466.015941:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941466.015959:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962b00. 00010000:00000001:3.0:1773941466.015967:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941466.015973:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485c00. 00000020:00000001:3.0:1773941466.015978:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941466.015981:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485c00 with handle 0xa33a5be14eae1d16 to hash 00000020:00000001:3.0:1773941466.015983:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941466.015984:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860128768 : -128249849422848 : ffff8b5b81485c00) 00010000:00000001:3.0:1773941466.015987:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860128768 : -128249849422848 : ffff8b5b81485c00) 00010000:00000001:3.0:1773941466.015991:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941466.015997:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941466.016002:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485c00/0xa33a5be14eae1d16 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941466.016014:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485c00/0xa33a5be14eae1d16 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941466.016025:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485c00/0xa33a5be14eae1d16 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941466.016031:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941466.016034:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941466.016042:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e9c0. 00000100:00000001:3.0:1773941466.016044:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387402176 : -128249322149440 : ffff8b5ba0b5e9c0) 00000100:00000001:3.0:1773941466.016047:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941466.016049:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109632384 : -128249599919232 : ffff8b5b90277b80) 00000100:00000001:3.0:1773941466.016056:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941466.016062:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941466.016066:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941466.016074:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941466.016090:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90277b80 x1860112241153664/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941466.016101:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941466.016104:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941466.016108:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941466.016111:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941466.016114:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941466.016116:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241153664:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941466.016120:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941466.016126:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941466.016129:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941466.016132:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941466.016137:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90262000. 02000000:00000001:3.0:1773941466.016146:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941466.016156:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941466.016161:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab3c8. 00000400:00000010:3.0:1773941466.016170:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a000. 00000100:00000200:3.0:1773941466.016178:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241153664, portal 25 00000100:00000001:3.0:1773941466.016179:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941466.016180:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109632384 : -128249599919232 : ffff8b5b90277b80) 00000100:00000040:3.0:1773941466.016259:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90277b80 x1860112241153664/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941482 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941466.016266:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941466.016269:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941466.016271:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6ab28. 00000100:00000200:3.0:1773941466.016274:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241153664, offset 0 00000400:00000200:3.0:1773941466.016280:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941466.016288:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941466.016296:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941466.016301:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941466.016305:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941466.016308:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941466.016315:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941466.016317:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[74, 73] 00000400:00000200:3.0:1773941466.016320:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941466.016323:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [180:180:256:4294967295] 192.168.204.153@tcp LPNI seq info [75:75:8:4294967295] 00000400:00000200:3.0:1773941466.016327:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941466.016336:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941466.016342:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941466.016346:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69c00. 00000800:00000200:3.0:1773941466.016353:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941466.016360:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941466.016365:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69c00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941466.016380:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941466.016384:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941466.016386:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294906379 00000100:00000001:3.0:1773941466.016391:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294904461 00000100:00000001:3.0:1773941466.016394:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294906379 00000100:00000001:3.0:1773941466.016398:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294904461 00000100:00000001:3.0:1773941466.016400:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941466.016402:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941466.016404:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941466.016406:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941466.016424:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941466.016428:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941466.016430:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941466.016498:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941466.016504:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69c00. 00000400:00000200:2.0:1773941466.016511:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941466.016518:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941466.016525:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6ab28 00000400:00000010:2.0:1773941466.016529:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6ab28. 00000100:00000001:2.0:1773941466.016535:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941466.016542:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277b80 x1860112241153664/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941482 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941466.016555:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941466.016560:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90277b80 x1860112241153664/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941482 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941466.016570:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941466.016572:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941466.017088:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941466.017095:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941466.017098:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941466.017100:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941466.017105:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941466.017109:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941466.017153:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941466.017162:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941466.017166:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941466.017170:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605680 00000400:00000200:3.0:1773941466.017173:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941466.017177:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xe4d [1] + 224 00000800:00000001:3.0:1773941466.017183:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941466.017189:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941466.017191:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941466.017193:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941466.017197:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941466.017202:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277b80 x1860112241153664/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941482 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941466.017210:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90277b80 x1860112241153664/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941482 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941466.017225:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941466.017231:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941466.017234:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941466.017283:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941466.017290:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941466.017292:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941466.017295:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab3c8. 00000400:00000200:1.0:1773941466.017299:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a000 00000400:00000010:1.0:1773941466.017300:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a000. 00000100:00000001:1.0:1773941466.017302:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941466.017307:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277b80 x1860112241153664/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941482 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941466.017315:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277b80 x1860112241153664/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941482 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941466.017320:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941466.017329:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941466.017331:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941466.017334:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941466.017337:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941466.017339:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941466.017341:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941466.017347:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=45 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941466.017349:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941466.017352:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941466.017356:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941466.017358:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941466.017361:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941466.017363:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941466.017367:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941466.017371:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941466.017374:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941466.017376:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941466.017377:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941466.017379:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941466.017382:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90277b80 x1860112241153664/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941482 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941466.017387:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941466.017388:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941466.017389:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941466.017392:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941466.017395:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941466.017396:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941466.017397:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941466.017398:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941466.017400:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941466.017401:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941466.017402:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941466.017404:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941466.017405:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941466.017520:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941466.017523:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 234 00000400:00000200:1.0:1773941466.017527:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941466.017532:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941466.017535:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941466.017538:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941466.017540:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941466.017542:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941466.017545:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941466.017546:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941466.017548:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941466.017550:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941466.017551:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941466.017553:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941466.017555:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941466.017556:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941466.017558:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941466.017559:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941466.017562:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941466.017565:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941466.017566:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941466.017568:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941466.017571:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941466.017573:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941466.017574:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941466.017576:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941466.017580:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61500. 00000020:00000040:1.0:1773941466.017581:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941466.017584:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941466.017585:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941466.017587:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941466.017589:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941466.017595:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941466.017599:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0736c00. 00000100:00000001:1.0:1773941466.017602:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941466.017607:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d61500 x1860112241153792/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941466.017614:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941466.017616:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941466.017619:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000bdbed3f] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d61500 x1860112241153792/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941466.017637:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941466.017640:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941466.017641:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941466.017643:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941466.017645:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941466.017646:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941466.017648:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941466.017649:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941466.017651:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941466.017655:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90277b80 x1860112241153664/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941482 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941466.017660:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941466.017663:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941466.017665:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941466.017667:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941466.017670:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90277b80 x1860112241153664/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941482 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941466.017672:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941466.017679:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1773941466.017679:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241153664:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941466.017685:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941466.017686:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941466.017690:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941466.017692:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000040:2.0:1773941466.017697:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d61500 x1860112241153792/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941466.017698:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941466.017725:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90277b80 x1860112241153664/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941482 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941466.017732:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941466.017734:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e9c0. 00000100:00000001:1.0:1773941466.017737:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941466.017739:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941466.017744:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941466.017748:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:2.0:1773941466.017752:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1773941466.017752:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941466.017755:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485c00 refcount=4 00000100:00000001:2.0:1773941466.017756:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941466.017758:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860128768 : -128249849422848 : ffff8b5b81485c00) 00010000:00000001:1.0:1773941466.017761:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860128768 : -128249849422848 : ffff8b5b81485c00) 02000000:00000001:2.0:1773941466.017762:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941466.017767:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00010000:1.0:1773941466.017768:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485c00/0xa33a5be14eae1d16 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941466.017774:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941466.017780:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00100000:2.0:1773941466.017784:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241153792:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00010000:1.0:1773941466.017786:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485c00/0xa33a5be14eae1d16 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941466.017796:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941466.017798:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000001:2.0:1773941466.017801:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00010000:1.0:1773941466.017802:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485c00/0xa33a5be14eae1d16 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941466.017806:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941466.017811:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000001:2.0:1773941466.017812:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941466.017813:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941466.017817:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485c00/0xa33a5be14eae1d16 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:2.0:1773941466.017820:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce7c00. 00010000:00000001:1.0:1773941466.017821:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941466.017823:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941466.017827:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941466.017828:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941466.017838:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485c00/0xa33a5be14eae1d16 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941466.017844:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941466.017849:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485c00 already unused, calling callback (ffffffffc1482c20) 00000400:00000010:2.0:1773941466.017852:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 10000000:00000001:1.0:1773941466.017853:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941466.017856:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485c00/0xa33a5be14eae1d16 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941466.017860:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00000001:1.0:1773941466.017862:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941466.017865:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941466.017866:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:2.0:1773941466.017867:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241153792, portal 25 00000020:00000040:1.0:1773941466.017868:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485c00 refcount=5 00000100:00000001:2.0:1773941466.017872:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941466.017874:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121074944 : -128249588476672 : ffff8b5b90d61500) 00000020:00000001:1.0:1773941466.017881:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860128768 : -128249849422848 : ffff8b5b81485c00) 00000100:00000040:2.0:1773941466.017883:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d61500 x1860112241153792/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941476 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941466.017889:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860128768 : -128249849422848 : ffff8b5b81485c00) 00010000:00000001:1.0:1773941466.017895:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941466.017899:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485c00/0xa33a5be14eae1d16 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941466.017900:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941466.017908:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 10000000:00000001:1.0:1773941466.017911:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000010:2.0:1773941466.017913:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34b28. 10000000:00010000:1.0:1773941466.017915:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485c00/0xa33a5be14eae1d16 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941466.017921:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241153792, offset 0 10000000:01000000:1.0:1773941466.017922:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941466.017928:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941466.017929:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941466.017933:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:2.0:1773941466.017934:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941466.017936:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941466.017941:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:2.0:1773941466.017943:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941466.017944:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941466.017947:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941466.017950:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941466.017953:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000020:00000040:1.0:1773941466.017953:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485c00 with handle 0xa33a5be14eae1d16 from hash 00010000:00000001:1.0:1773941466.017958:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:2.0:1773941466.017961:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941466.017965:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941466.017968:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941466.017969:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941466.017970:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941466.017973:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941466.017976:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:2.0:1773941466.017977:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941466.017980:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941466.017983:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941466.017985:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941466.017986:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000001:1.0:1773941466.017988:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941466.017992:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485c00/0xa33a5be14eae1d16 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941466.017994:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[74, 75] 00010000:00000001:1.0:1773941466.018000:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941466.018002:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941466.018004:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000400:00000200:2.0:1773941466.018007:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941466.018007:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941466.018010:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941466.018012:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941466.018015:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941466.018016:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [181:181:256:4294967295] 192.168.204.143@tcp LPNI seq info [76:76:8:4294967295] 00010000:00010000:1.0:1773941466.018018:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485c00/0xa33a5be14eae1d16 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941466.018027:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000040:1.0:1773941466.018030:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962b00 count: 0 00000400:00000200:2.0:1773941466.018039:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941466.018041:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941466.018044:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941466.018047:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000800:00000200:2.0:1773941466.018049:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000100:00000040:1.0:1773941466.018054:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90277b80 x1860112241153664/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941482 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000800:00000010:2.0:1773941466.018058:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0a00. 00000100:00000001:1.0:1773941466.018062:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941466.018066:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000200:2.0:1773941466.018070:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 02000000:00000010:1.0:1773941466.018070:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90262000. 02000000:00000001:1.0:1773941466.018077:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941466.018079:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941466.018081:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941466.018086:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000800:00000200:2.0:1773941466.018087:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 02000000:00000010:1.0:1773941466.018088:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e3000. 00000800:00000200:2.0:1773941466.018092:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0a00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941466.018108:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941466.018114:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:2.0:1773941466.018117:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 02000000:00000001:1.0:1773941466.018118:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941466.018120:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277b80. 00000100:00000001:1.0:1773941466.018123:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941466.018124:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941466.018126:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941466.018132:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294906381 10000000:01000000:1.0:1773941466.018132:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941466.018136:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:2.0:1773941466.018140:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294906379 10000000:00000010:1.0:1773941466.018141:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0e80. 00000100:00000001:2.0:1773941466.018147:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294906381 00000020:00000010:1.0:1773941466.018148:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000100:00000001:2.0:1773941466.018154:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294906379 00000020:00000010:1.0:1773941466.018158:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000100:00000001:2.0:1773941466.018159:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000040:00000010:1.0:1773941466.018164:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000100:00000001:2.0:1773941466.018165:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941466.018171:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941466.018172:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000100:00000001:2.0:1773941466.018178:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1773941466.018179:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941466.018186:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f9480. 00000100:00000001:2.0:1773941466.018187:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941466.018191:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00040000:00000010:1.0:1773941466.018191:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 00000800:00000200:3.0:1773941466.018195:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 20000000:00000010:1.0:1773941466.018197:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 00000100:00000001:2.0:1773941466.018198:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:3.0:1773941466.018200:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0a00. 00000100:00000001:2.0:1773941466.018202:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000040:1.0:1773941466.018205:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000100:00000001:2.0:1773941466.018207:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941466.018208:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000001:1.0:1773941466.018209:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941466.018214:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941466.018217:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000400:00000200:3.0:1773941466.018222:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000040:00000010:1.0:1773941466.018223:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000400:00000200:3.0:1773941466.018229:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34b28 00000020:00000010:1.0:1773941466.018229:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000400:00000010:3.0:1773941466.018234:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34b28. 00000020:00000010:1.0:1773941466.018234:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941466.018239:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f9480. 00000100:00000001:3.0:1773941466.018242:0:2347:0:(events.c:34:request_out_callback()) Process entered 00040000:00000010:1.0:1773941466.018243:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941466.018248:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941466.018252:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 00000100:00000200:3.0:1773941466.018253:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61500 x1860112241153792/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941476 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000010:1.0:1773941466.018256:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0e80. 10000000:01000000:1.0:1773941466.018259:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941466.018261:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941466.018262:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941466.018264:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941466.018265:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941466.018266:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941466.018271:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:3.0:1773941466.018329:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941466.018333:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d61500 x1860112241153792/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941476 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941466.018337:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941466.018339:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941466.019793:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941466.019800:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941466.019803:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941466.019806:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941466.019811:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941466.019816:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941466.019822:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941466.019825:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941466.019829:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941466.019832:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605700 00000400:00000200:3.0:1773941466.019837:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941466.019901:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xe5d [1] + 0 00000800:00000001:3.0:1773941466.019907:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941466.019913:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941466.019914:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941466.019917:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941466.019920:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941466.019924:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61500 x1860112241153792/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941476 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941466.019932:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d61500 x1860112241153792/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941476 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941466.019944:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941466.019948:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941466.019950:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941466.019981:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941466.019985:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941466.019988:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941466.019990:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941466.019992:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941466.019997:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941466.019999:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941466.020002:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941466.020009:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61500 x1860112241153792/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941476 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941466.020020:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61500 x1860112241153792/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941476 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941466.020028:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941466.020030:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941466.020033:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941466.020035:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941466.020036:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941466.020038:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941466.020041:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941466.020044:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=45 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941466.020048:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941466.020057:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941466.020060:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941466.020062:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941466.020065:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941466.020068:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941466.020074:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d61500 x1860112241153792/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941476 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941466.020082:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941466.020085:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941466.020087:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941466.020089:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941466.020095:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941466.020098:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941466.020099:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941466.020101:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941466.020103:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941466.020105:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941466.020108:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941466.020113:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941466 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941466.020116:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941466.020123:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941466.020125:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941466.020127:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941466.020129:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941466.020131:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941466.020135:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941466.020138:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941466.020141:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941466.020145:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941466.020148:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941466.020151:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941466.020312:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941466.020316:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941466.020322:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d61500 x1860112241153792/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941476 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941466.020331:0:12191:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:0.0:1773941466.020331:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241153792:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941466.020335:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941466.020337:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1773941466.020349:0:12191:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000040:0.0:1773941466.020350:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d61500 x1860112241153792/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941476 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941466.020358:0:12191:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941466.020360:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941466.020363:0:12191:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:0.0:1773941466.020363:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941466.020367:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce7c00. 10000000:01000000:2.0:1773941466.020368:0:12191:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:0.0:1773941466.020371:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941466.020374:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:2.0:1773941466.020375:0:12191:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000040:0.0:1773941466.020375:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941466.020380:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:2.0:1773941466.020382:0:12191:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:0.0:1773941466.020382:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0736c00. 02000000:00000001:0.0:1773941466.020386:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941466.020388:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941466.020390:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61500. 00000001:00000001:2.0:1773941466.020391:0:12191:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941466.020393:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:2.0:1773941466.020395:0:12191:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:0.0:1773941466.020396:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:2.0:1773941466.020401:0:12191:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:0.0:1773941466.020401:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:2.0:1773941466.020407:0:12191:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941466.020408:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941466.020422:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941466.020424:0:12191:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941466.020426:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941466.020429:0:12191:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941466.020430:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941466.020435:0:12191:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941466.020439:0:12191:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941466.020442:0:12191:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941466.020444:0:12191:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941466.020448:0:12191:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941466.020451:0:12191:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941466.020452:0:12191:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941466.020454:0:12191:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:2.0:1773941466.020469:0:12191:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941466.020470:0:12191:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941466.020472:0:12191:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941466.020476:0:12191:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941466.020481:0:12191:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941466.020484:0:12191:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941466.020486:0:12191:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941466.020489:0:12191:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941466.020492:0:12191:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941466.025737:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962b00. 00010000:00000010:1.1:1773941466.025745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485c00. 00000020:00000001:0.0:1773941466.975908:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941466.975918:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941466.975924:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941467.807842:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941467.807852:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941467.807857:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941467.807859:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941467.807861:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941467.807862:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941467.807866:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941467.807868:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941467.807870:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941467.807874:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941467.807876:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941467.807892:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60a80. 00000020:00000040:1.0:1773941467.807898:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941467.807905:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941467.807909:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941467.807916:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941467.807926:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941467.807935:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941467.807942:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941467.807946:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941467.807955:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec500. 00010000:00000001:1.0:1773941467.807961:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941467.807964:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941467.807968:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941467.807971:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1d1d to hash 00000020:00000001:1.0:1773941467.807973:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941467.807975:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941467.807977:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941467.807982:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941467.807985:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941467.807988:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d1d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941467.808003:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d1d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941467.808031:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d1d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941467.808036:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941467.808038:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941467.808049:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941467.808051:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941467.808053:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941467.808055:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121072256 : -128249588479360 : ffff8b5b90d60a80) 00000100:00000001:1.0:1773941467.808060:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941467.808064:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941467.808066:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941467.808068:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941467.808079:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60a80 x1860112241153920/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941467.808087:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941467.808089:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941467.808092:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941467.808095:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941467.808097:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941467.808099:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241153920:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941467.808103:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941467.808109:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941467.808111:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941467.808113:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941467.808117:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734c00. 02000000:00000001:1.0:1773941467.808118:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941467.808127:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941467.808130:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941467.808137:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941467.808144:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241153920, portal 25 00000100:00000001:1.0:1773941467.808146:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941467.808150:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121072256 : -128249588479360 : ffff8b5b90d60a80) 00000100:00000040:1.0:1773941467.808154:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60a80 x1860112241153920/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941483 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941467.808161:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941467.808163:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941467.808165:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1880. 00000100:00000200:1.0:1773941467.808168:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241153920, offset 0 00000400:00000200:1.0:1773941467.808174:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941467.808183:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941467.808192:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941467.808198:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941467.808202:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941467.808206:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941467.808208:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941467.808211:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[76, 75] 00000400:00000200:1.0:1773941467.808214:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941467.808217:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [182:182:256:4294967295] 192.168.204.153@tcp LPNI seq info [77:77:8:4294967295] 00000400:00000200:1.0:1773941467.808221:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941467.808230:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941467.808236:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941467.808241:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024be00. 00000800:00000200:1.0:1773941467.808248:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941467.808255:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941467.808260:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024be00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941467.808279:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941467.808283:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941467.808285:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294908171 00000100:00000001:1.0:1773941467.808292:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294906381 00000100:00000001:1.0:1773941467.808297:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294908171 00000100:00000001:1.0:1773941467.808299:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294906381 00000100:00000001:1.0:1773941467.808306:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941467.808308:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941467.808311:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941467.808313:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941467.808315:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941467.808318:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941467.808322:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941467.808440:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941467.808446:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024be00. 00000400:00000200:3.0:1773941467.808452:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941467.808458:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941467.808462:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1880 00000400:00000010:3.0:1773941467.808465:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1880. 00000100:00000001:3.0:1773941467.808469:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941467.808473:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60a80 x1860112241153920/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941483 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941467.808492:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941467.808495:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d60a80 x1860112241153920/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941483 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941467.808501:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941467.808502:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941467.809176:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941467.809190:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941467.809193:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941467.809197:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941467.809203:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941467.809209:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941467.809216:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941467.809269:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941467.809275:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941467.809280:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605780 00000400:00000200:3.0:1773941467.809285:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941467.809291:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xe6d [1] + 224 00000800:00000001:3.0:1773941467.809299:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941467.809309:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941467.809312:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941467.809316:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941467.809321:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941467.809326:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60a80 x1860112241153920/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941483 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941467.809338:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d60a80 x1860112241153920/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941483 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941467.809357:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941467.809366:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941467.809371:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941467.809394:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941467.809400:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941467.809402:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941467.809404:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941467.809408:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941467.809409:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941467.809411:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941467.809415:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60a80 x1860112241153920/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941483 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941467.809423:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60a80 x1860112241153920/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941483 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941467.809428:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941467.809429:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941467.809431:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941467.809434:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941467.809436:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941467.809438:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941467.809440:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941467.809446:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=46 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941467.809449:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941467.809452:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941467.809456:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941467.809458:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941467.809461:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941467.809463:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941467.809467:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941467.809471:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941467.809479:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941467.809481:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941467.809483:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941467.809485:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941467.809487:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d60a80 x1860112241153920/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941483 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941467.809493:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941467.809494:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941467.809496:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941467.809498:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941467.809501:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941467.809502:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941467.809504:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941467.809504:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941467.809505:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941467.809506:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941467.809507:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941467.809509:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941467.809510:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941467.809513:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941467.809515:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 236 00000400:00000200:1.0:1773941467.809518:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941467.809521:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941467.809525:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941467.809528:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941467.809530:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941467.809532:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941467.809534:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941467.809536:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941467.809537:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941467.809539:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941467.809541:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941467.809542:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941467.809550:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941467.809552:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941467.809553:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941467.809555:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941467.809557:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941467.809560:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941467.809561:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941467.809563:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941467.809566:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941467.809568:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941467.809570:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941467.809571:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941467.809575:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61c00. 00000020:00000040:1.0:1773941467.809576:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941467.809579:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941467.809580:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941467.809582:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941467.809584:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941467.809585:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941467.809589:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0735c00. 00000100:00000001:1.0:1773941467.809592:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941467.809596:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d61c00 x1860112241154048/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941467.809603:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941467.809605:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941467.809609:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbfa947b] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d61c00 x1860112241154048/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941467.809628:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941467.809631:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941467.809632:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941467.809634:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941467.809636:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941467.809641:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941467.809642:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941467.809644:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941467.809645:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941467.809649:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60a80 x1860112241153920/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941483 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941467.809654:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941467.809656:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941467.809659:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941467.809661:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941467.809664:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60a80 x1860112241153920/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941483 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941467.809670:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241153920:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941467.809674:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941467.809676:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941467.809677:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941467.809681:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941467.809682:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941467.809683:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941467.809686:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941467.809688:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60a80 x1860112241153920/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941483 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941467.809695:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d61c00 x1860112241154048/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941467.809697:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941467.809701:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941467.809723:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941467.809724:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941467.809727:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941467.809730:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941467.809732:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941467.809733:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 00000020:00000001:1.0:1773941467.809738:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000001:0.0:1773941467.809741:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:1.0:1773941467.809742:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000001:0.0:1773941467.809744:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941467.809747:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d1d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941467.809748:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941467.809751:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941467.809755:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941467.809761:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241154048:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941467.809765:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941467.809766:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00010000:1.0:1773941467.809770:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d1d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941467.809774:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941467.809778:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d1d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941467.809778:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941467.809781:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941467.809785:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941467.809787:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941467.809790:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941467.809791:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d1d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:0.0:1773941467.809792:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0000. 02000000:00000001:0.0:1773941467.809798:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941467.809800:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:0.0:1773941467.809803:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941467.809805:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941467.809809:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000010:0.0:1773941467.809809:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00010000:1.0:1773941467.809813:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d1d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941467.809817:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00010000:00010000:1.0:1773941467.809818:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941467.809822:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000200:0.0:1773941467.809824:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241154048, portal 25 10000000:00010000:1.0:1773941467.809825:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d1d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941467.809831:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941467.809834:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:0.0:1773941467.809835:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121076736 : -128249588474880 : ffff8b5b90d61c00) 00010000:00000001:1.0:1773941467.809837:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941467.809839:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941467.809842:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000020:00000001:1.0:1773941467.809844:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941467.809846:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000040:0.0:1773941467.809848:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d61c00 x1860112241154048/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941467.809857:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941467.809861:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d1d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941467.809864:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941467.809871:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 10000000:00000001:1.0:1773941467.809872:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941467.809875:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d1d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941467.809876:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6ca18. 10000000:01000000:1.0:1773941467.809883:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000200:0.0:1773941467.809883:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241154048, offset 0 10000000:00000040:1.0:1773941467.809886:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941467.809889:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941467.809892:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941467.809895:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:0.0:1773941467.809896:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941467.809899:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941467.809901:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941467.809903:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:0.0:1773941467.809904:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941467.809906:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941467.809910:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1d1d from hash 00010000:00000001:1.0:1773941467.809913:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:0.0:1773941467.809913:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:1.0:1773941467.809914:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941467.809917:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941467.809920:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941467.809921:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:0.0:1773941467.809922:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941467.809923:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941467.809926:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941467.809928:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:0.0:1773941467.809929:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941467.809930:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941467.809934:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941467.809937:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:1.0:1773941467.809938:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d1d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941467.809944:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941467.809947:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941467.809950:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941467.809952:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000400:00000200:0.0:1773941467.809953:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[76, 77] 00010000:00000001:1.0:1773941467.809954:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941467.809961:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941467.809961:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941467.809964:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941467.809966:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941467.809970:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d1d lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941467.809971:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [183:183:256:4294967295] 192.168.204.143@tcp LPNI seq info [78:78:8:4294967295] 00010000:00000040:1.0:1773941467.809985:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec500 count: 0 00000400:00000200:0.0:1773941467.809985:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941467.809999:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941467.810002:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941467.810004:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941467.810006:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941467.810009:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60a80 x1860112241153920/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941483 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941467.810019:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941467.810022:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941467.810025:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734c00. 00000800:00000200:0.0:1773941467.810026:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 02000000:00000001:1.0:1773941467.810029:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941467.810032:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941467.810034:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000800:00000010:0.0:1773941467.810035:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b91e1e300. 00000020:00000001:1.0:1773941467.810038:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941467.810041:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 00000800:00000200:0.0:1773941467.810042:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 02000000:00000001:1.0:1773941467.810045:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941467.810047:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941467.810048:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60a80. 00000100:00000001:1.0:1773941467.810053:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941467.810055:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941467.810057:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941467.810061:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000800:00000200:0.0:1773941467.810061:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 10000000:00000001:1.0:1773941467.810066:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000800:00000200:0.0:1773941467.810068:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b91e1e300 type 1, nob 616 niov 1 nkiov 1 10000000:00000010:1.0:1773941467.810070:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0e80. 00000020:00000010:1.0:1773941467.810075:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941467.810079:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000100:00000001:0.0:1773941467.810083:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:1.0:1773941467.810084:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941467.810091:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000400:00000200:0.0:1773941467.810092:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941467.810096:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294908173 00000020:00000010:1.0:1773941467.810097:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941467.810104:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000001:0.0:1773941467.810106:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294908171 00040000:00000010:1.0:1773941467.810114:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 00000100:00000001:0.0:1773941467.810115:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294908173 20000000:00000010:1.0:1773941467.810121:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 00000100:00000001:0.0:1773941467.810123:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294908171 10000000:00000040:1.0:1773941467.810127:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000100:00000001:0.0:1773941467.810129:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:00000001:1.0:1773941467.810131:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941467.810137:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941467.810139:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000100:00000001:0.0:1773941467.810142:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:1.0:1773941467.810144:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000100:00000001:0.0:1773941467.810148:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1773941467.810149:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941467.810156:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 00000100:00000001:0.0:1773941467.810158:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 80000000:00000010:1.0:1773941467.810159:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00000800:00000200:3.0:1773941467.810163:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1773941467.810163:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00040000:00000010:1.0:1773941467.810165:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 00000100:00000001:0.0:1773941467.810168:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 20000000:00000010:1.0:1773941467.810170:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000100:00000001:0.0:1773941467.810173:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000010:1.0:1773941467.810176:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 00000800:00000010:3.0:1773941467.810177:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b91e1e300. 00000100:00000001:0.0:1773941467.810177:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000010:1.0:1773941467.810179:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0e80. 00000100:00000001:0.0:1773941467.810185:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941467.810186:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:01000000:1.0:1773941467.810186:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941467.810193:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941467.810197:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000400:00000200:3.0:1773941467.810199:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 10000000:00000001:1.0:1773941467.810199:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941467.810202:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941467.810205:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 00000400:00000200:3.0:1773941467.810207:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6ca18 10000000:00000001:1.0:1773941467.810208:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000400:00000010:3.0:1773941467.810212:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6ca18. 00000100:00000001:3.0:1773941467.810218:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941467.810223:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61c00 x1860112241154048/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941477 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941467.810233:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941467.810289:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d61c00 x1860112241154048/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941477 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941467.810298:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941467.810300:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941467.811442:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941467.811450:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941467.811453:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941467.811456:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941467.811461:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941467.811466:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941467.811470:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941467.811473:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941467.811476:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941467.811479:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605800 00000400:00000200:3.0:1773941467.811481:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941467.811485:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xe75 [1] + 0 00000800:00000001:3.0:1773941467.811489:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941467.811545:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941467.811548:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941467.811551:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941467.811554:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941467.811559:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61c00 x1860112241154048/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941477 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941467.811567:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d61c00 x1860112241154048/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941477 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941467.811588:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941467.811592:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941467.811615:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941467.811632:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941467.811636:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941467.811641:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941467.811643:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941467.811645:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941467.811650:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941467.811653:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941467.811656:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941467.811662:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61c00 x1860112241154048/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941477 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941467.811673:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61c00 x1860112241154048/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941477 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941467.811682:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941467.811684:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941467.811686:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941467.811695:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941467.811698:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941467.811700:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941467.811728:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941467.811734:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=46 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941467.811739:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941467.811742:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941467.811744:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941467.811748:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941467.811751:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941467.811754:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941467.811760:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d61c00 x1860112241154048/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941477 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941467.811769:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941467.811773:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941467.811774:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941467.811777:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941467.811784:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941467.811787:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941467.811788:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941467.811791:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941467.811793:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941467.811795:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941467.811798:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941467.811803:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941467 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941467.811806:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941467.811814:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941467.811816:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941467.811818:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941467.811820:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941467.811822:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941467.811826:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941467.811829:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941467.811832:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941467.811842:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941467.811846:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941467.811849:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941467.812095:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941467.812100:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941467.812109:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d61c00 x1860112241154048/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941477 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941467.812116:0:12192:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941467.812124:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241154048:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00080000:0.0:1773941467.812129:0:12192:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941467.812135:0:12192:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941467.812146:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941467.812149:0:12192:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941467.812150:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:01000000:0.0:1773941467.812155:0:12192:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000040:2.0:1773941467.812159:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d61c00 x1860112241154048/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941477 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:0.0:1773941467.812161:0:12192:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941467.812165:0:12192:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941467.812171:0:12192:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941467.812172:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941467.812174:0:12192:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941467.812176:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941467.812177:0:12192:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941467.812181:0:12192:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941467.812183:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0000. 10000000:00000001:0.0:1773941467.812183:0:12192:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941467.812186:0:12192:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:2.0:1773941467.812191:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:0.0:1773941467.812193:0:12192:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000020:00000001:2.0:1773941467.812195:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941467.812196:0:12192:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000040:2.0:1773941467.812199:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941467.812200:0:12192:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941467.812204:0:12192:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941467.812205:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:0.0:1773941467.812207:0:12192:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 02000000:00000010:2.0:1773941467.812209:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0735c00. 00000100:00000001:0.0:1773941467.812211:0:12192:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941467.812214:0:12192:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 02000000:00000001:2.0:1773941467.812216:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000040:0.0:1773941467.812216:0:12192:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 02000000:00000001:2.0:1773941467.812221:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941467.812225:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61c00. 00000100:00000001:2.0:1773941467.812229:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941467.812231:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941467.812234:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941467.812236:0:12192:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:2.0:1773941467.812239:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000001:0.0:1773941467.812240:0:12192:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941467.812243:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941467.812245:0:12192:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941467.812249:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941467.812254:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:02000000:0.0:1773941467.812254:0:12192:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941467.812263:0:12192:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941467.812267:0:12192:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941467.812269:0:12192:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941467.812271:0:12192:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941467.812273:0:12192:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941467.815741:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec500. 00010000:00000010:1.1:1773941467.815749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:3.0:1773941467.999944:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941467.999956:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941467.999969:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941468.063885:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (243) 10000000:00000001:1.0:1773941468.959865:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941468.959878:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941468.959885:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941468.959888:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941468.959891:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941468.959892:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941468.959904:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941468.959907:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941468.959910:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941468.959914:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941468.959916:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941468.959930:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60a80. 00000020:00000040:1.0:1773941468.959936:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941468.959941:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941468.959964:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941468.959971:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941468.959980:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941468.959990:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941468.959996:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941468.960000:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941468.960011:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec500. 00010000:00000001:1.0:1773941468.960018:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941468.960023:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941468.960028:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941468.960031:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1d24 to hash 00000020:00000001:1.0:1773941468.960043:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941468.960046:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941468.960050:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941468.960054:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941468.960059:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941468.960064:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d24 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941468.960078:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d24 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941468.960093:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d24 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941468.960110:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941468.960113:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941468.960122:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941468.960125:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941468.960128:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941468.960130:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121072256 : -128249588479360 : ffff8b5b90d60a80) 00000100:00000001:1.0:1773941468.960136:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941468.960143:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941468.960145:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941468.960158:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941468.960172:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60a80 x1860112241154176/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941468.960183:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941468.960187:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941468.960192:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941468.960195:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941468.960198:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941468.960202:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241154176:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941468.960208:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941468.960215:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941468.960218:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941468.960221:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941468.960226:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734c00. 02000000:00000001:1.0:1773941468.960229:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941468.960239:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941468.960245:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941468.960254:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941468.960263:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241154176, portal 25 00000100:00000001:1.0:1773941468.960265:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941468.960267:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121072256 : -128249588479360 : ffff8b5b90d60a80) 00000100:00000040:1.0:1773941468.960273:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60a80 x1860112241154176/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941484 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941468.960282:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941468.960285:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941468.960289:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1ee0. 00000100:00000200:1.0:1773941468.960292:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241154176, offset 0 00000400:00000200:1.0:1773941468.960298:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941468.960309:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941468.960319:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941468.960325:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941468.960330:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941468.960336:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941468.960346:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941468.960349:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[78, 77] 00000400:00000200:1.0:1773941468.960383:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941468.960390:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [184:184:256:4294967295] 192.168.204.153@tcp LPNI seq info [79:79:8:4294967295] 00000400:00000200:1.0:1773941468.960396:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941468.960410:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941468.960417:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941468.960424:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b000. 00000800:00000200:1.0:1773941468.960433:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941468.960441:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941468.960447:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b000 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941468.960468:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941468.960472:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941468.960476:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294909323 00000100:00000001:1.0:1773941468.960484:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294908173 00000100:00000001:1.0:1773941468.960498:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294909323 00000100:00000001:1.0:1773941468.960503:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294908173 00000100:00000001:1.0:1773941468.960507:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941468.960511:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941468.960515:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941468.960517:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941468.960521:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941468.960525:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941468.960529:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941468.960655:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941468.960666:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b000. 00000400:00000200:3.0:1773941468.960674:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941468.960684:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941468.960692:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1ee0 00000400:00000010:3.0:1773941468.960696:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1ee0. 00000100:00000001:3.0:1773941468.960701:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941468.960827:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60a80 x1860112241154176/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941484 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941468.960842:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941468.960846:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d60a80 x1860112241154176/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941484 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941468.960864:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941468.960867:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941468.976176:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941468.976234:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941468.976239:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941468.976242:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941468.976252:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941468.976260:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941468.976269:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941468.976274:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941468.976280:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941468.976284:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605880 00000400:00000200:3.0:1773941468.976290:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941468.976297:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xe7d [1] + 224 00000800:00000001:3.0:1773941468.976322:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941468.976341:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941468.976345:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941468.976350:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941468.976369:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941468.976379:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60a80 x1860112241154176/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941484 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941468.976393:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d60a80 x1860112241154176/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941484 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941468.976417:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941468.976425:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941468.976429:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941468.976467:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941468.976475:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941468.976476:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941468.976479:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941468.976483:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941468.976485:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941468.976488:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941468.976498:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60a80 x1860112241154176/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941484 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941468.976506:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60a80 x1860112241154176/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941484 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941468.976511:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941468.976512:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941468.976514:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941468.976517:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941468.976519:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941468.976520:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941468.976523:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941468.976528:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=47 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941468.976530:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941468.976532:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941468.976535:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941468.976538:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941468.976540:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941468.976543:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941468.976546:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941468.976549:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941468.976551:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941468.976553:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941468.976554:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941468.976556:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941468.976558:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d60a80 x1860112241154176/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941484 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941468.976564:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941468.976564:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941468.976566:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941468.976568:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941468.976570:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941468.976571:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941468.976576:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941468.976577:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941468.976578:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941468.976579:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941468.976580:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941468.976582:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941468.976584:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941468.976586:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941468.976588:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 238 00000400:00000200:1.0:1773941468.976591:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941468.976595:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941468.976599:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941468.976601:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941468.976604:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941468.976606:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941468.976610:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941468.976612:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941468.976615:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941468.976617:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941468.976620:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941468.976622:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941468.976625:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941468.976627:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941468.976630:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941468.976632:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941468.976635:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941468.976639:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941468.976642:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941468.976644:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941468.976648:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941468.976651:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941468.976653:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941468.976661:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941468.976666:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63b80. 00000020:00000040:1.0:1773941468.976670:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941468.976675:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941468.976677:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941468.976680:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941468.976682:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941468.976687:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941468.976694:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0735c00. 00000100:00000001:1.0:1773941468.976701:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941468.976745:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63b80 x1860112241154304/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941468.976753:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941468.976756:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941468.976760:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000957145d0] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63b80 x1860112241154304/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941468.976778:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941468.976780:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941468.976781:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941468.976784:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941468.976786:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941468.976788:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941468.976789:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941468.976791:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941468.976792:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941468.976796:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60a80 x1860112241154176/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941484 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941468.976801:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941468.976803:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941468.976806:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941468.976808:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941468.976813:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60a80 x1860112241154176/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941484 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941468.976836:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241154176:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941468.976840:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941468.976843:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941468.976849:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941468.976851:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941468.976853:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941468.976860:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941468.976864:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941468.976871:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63b80 x1860112241154304/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941468.976873:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60a80 x1860112241154176/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941484 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941468.976946:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941468.976946:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941468.976950:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941468.976952:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 02000000:00000001:2.0:1773941468.976957:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941468.976957:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941468.976962:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941468.976963:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941468.976979:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941468.976980:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941468.976987:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00100000:2.0:1773941468.976990:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241154304:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000020:00000001:1.0:1773941468.976991:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941468.976995:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 00000020:00000001:1.0:1773941468.977003:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000001:2.0:1773941468.977004:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941468.977008:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00010000:1.0:1773941468.977013:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d24 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941468.977023:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:2.0:1773941468.977025:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00010000:1.0:1773941468.977028:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d24 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941468.977029:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941468.977039:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941468.977046:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000010:2.0:1773941468.977048:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce4800. 00010000:00010000:1.0:1773941468.977049:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d24 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941468.977052:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941468.977068:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941468.977069:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941468.977073:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00010000:00000001:1.0:1773941468.977079:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941468.977081:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d24 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941468.977082:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00000001:1.0:1773941468.977094:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000200:2.0:1773941468.977095:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241154304, portal 25 00000100:00000001:2.0:1773941468.977102:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941468.977102:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941468.977109:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121084800 : -128249588466816 : ffff8b5b90d63b80) 00010000:00000001:1.0:1773941468.977111:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941468.977120:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d24 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941468.977126:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63b80 x1860112241154304/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941468.977146:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941468.977156:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:2.0:1773941468.977157:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 10000000:00010000:1.0:1773941468.977160:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d24 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941468.977165:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941468.977175:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f342a8. 00010000:00000001:1.0:1773941468.977177:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941468.977181:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000200:2.0:1773941468.977182:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241154304, offset 0 00000020:00000001:1.0:1773941468.977182:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941468.977188:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000020:00000001:1.0:1773941468.977192:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000400:00000200:2.0:1773941468.977198:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941468.977199:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000400:00000200:2.0:1773941468.977206:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941468.977211:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:2.0:1773941468.977215:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00010000:1.0:1773941468.977216:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d24 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941468.977224:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00000001:1.0:1773941468.977232:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941468.977233:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 10000000:00010000:1.0:1773941468.977237:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d24 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941468.977241:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941468.977250:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:01000000:1.0:1773941468.977258:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941468.977261:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[78, 79] 10000000:00000040:1.0:1773941468.977267:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941468.977273:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941468.977278:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00010000:1.0:1773941468.977279:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941468.977285:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941468.977293:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941468.977295:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941468.977297:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941468.977300:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [185:185:256:4294967295] 192.168.204.143@tcp LPNI seq info [80:80:8:4294967295] 00010000:00000001:1.0:1773941468.977300:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941468.977307:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1d24 from hash 00010000:00000001:1.0:1773941468.977312:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941468.977315:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941468.977316:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941468.977318:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941468.977320:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941468.977323:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941468.977325:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941468.977327:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941468.977328:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941468.977330:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941468.977336:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941468.977340:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d24 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941468.977351:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941468.977365:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941468.977369:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941468.977375:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941468.977376:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941468.977378:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941468.977381:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941468.977382:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941468.977385:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941468.977385:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d24 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:2.0:1773941468.977397:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0c00. 00010000:00000040:1.0:1773941468.977398:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec500 count: 0 00000800:00000200:2.0:1773941468.977413:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941468.977415:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941468.977417:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941468.977420:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941468.977429:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60a80 x1860112241154176/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941484 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941468.977439:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000800:00000200:2.0:1773941468.977441:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 02000000:00000001:1.0:1773941468.977443:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000200:2.0:1773941468.977448:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0c00 type 1, nob 616 niov 1 nkiov 1 02000000:00000010:1.0:1773941468.977452:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734c00. 02000000:00000001:1.0:1773941468.977456:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941468.977457:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941468.977459:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941468.977460:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941468.977462:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941468.977466:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1773941468.977468:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941468.977468:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941468.977471:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60a80. 00000400:00000200:2.0:1773941468.977475:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941468.977475:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941468.977478:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941468.977483:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294909340 10000000:00000001:1.0:1773941468.977484:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941468.977493:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941468.977496:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941468.977499:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0e80. 00000100:00000001:2.0:1773941468.977503:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294909323 00000020:00000010:1.0:1773941468.977503:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941468.977512:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000100:00000001:2.0:1773941468.977516:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294909340 00000040:00000010:1.0:1773941468.977518:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941468.977525:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000100:00000001:2.0:1773941468.977528:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294909323 00000020:00000010:1.0:1773941468.977529:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 00000100:00000001:2.0:1773941468.977541:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 80000000:00000010:1.0:1773941468.977541:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000001:2.0:1773941468.977553:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:1.0:1773941468.977555:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 00000800:00000200:3.0:1773941468.977560:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941468.977562:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 20000000:00000010:1.0:1773941468.977565:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 00000100:00000001:2.0:1773941468.977568:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:3.0:1773941468.977569:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0c00. 10000000:00000040:1.0:1773941468.977574:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000400:00000200:3.0:1773941468.977579:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000001:1.0:1773941468.977579:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941468.977588:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:1.0:1773941468.977588:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000400:00000200:3.0:1773941468.977590:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941468.977592:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941468.977600:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000040:00000010:1.0:1773941468.977602:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000400:00000200:3.0:1773941468.977603:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f342a8 00000100:00000001:2.0:1773941468.977604:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941468.977611:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941468.977612:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f342a8. 00000020:00000010:1.0:1773941468.977615:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000100:00000001:2.0:1773941468.977621:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941468.977624:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941468.977634:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63b80 x1860112241154304/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941468.977634:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941468.977643:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00000100:00000001:3.0:1773941468.977651:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00040000:00000010:1.0:1773941468.977653:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 00000100:00000040:3.0:1773941468.977659:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63b80 x1860112241154304/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941478 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 20000000:00000010:1.0:1773941468.977661:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941468.977667:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941468.977671:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0e80. 10000000:01000000:1.0:1773941468.977674:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941468.977683:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941468.977686:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941468.977689:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941468.977691:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941468.977693:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941468.977695:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:3.0:1773941468.977765:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941468.977767:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941468.978816:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941468.978823:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941468.978825:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941468.978828:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941468.978833:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941468.978838:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941468.978845:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941468.978896:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941468.978902:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941468.978906:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605900 00000400:00000200:3.0:1773941468.978910:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941468.978915:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xe85 [1] + 0 00000800:00000001:3.0:1773941468.978920:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941468.978926:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941468.978929:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941468.978933:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941468.978936:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941468.978941:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63b80 x1860112241154304/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941478 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941468.978953:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63b80 x1860112241154304/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941478 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941468.978970:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941468.978974:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941468.978977:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941468.979001:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941468.979006:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941468.979011:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941468.979013:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941468.979016:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941468.979023:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941468.979025:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941468.979029:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941468.979036:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63b80 x1860112241154304/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941478 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941468.979048:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63b80 x1860112241154304/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941478 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941468.979057:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941468.979059:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941468.979061:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941468.979070:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941468.979072:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941468.979074:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941468.979076:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941468.979082:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=47 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941468.979085:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941468.979087:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941468.979088:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941468.979091:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941468.979093:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941468.979095:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941468.979099:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63b80 x1860112241154304/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941478 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941468.979103:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941468.979105:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941468.979107:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941468.979109:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941468.979116:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941468.979118:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941468.979120:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941468.979122:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941468.979124:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941468.979126:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941468.979127:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941468.979131:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941468 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941468.979133:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941468.979139:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941468.979141:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941468.979143:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941468.979145:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941468.979146:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941468.979149:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941468.979152:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941468.979154:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941468.979162:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941468.979166:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941468.979169:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941468.979411:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941468.979415:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941468.979419:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63b80 x1860112241154304/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941478 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941468.979426:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241154304:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941468.979431:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941468.979433:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941468.979436:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63b80 x1860112241154304/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941478 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941468.979442:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941468.979444:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941468.979447:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce4800. 00000100:00000001:0.0:1773941468.979449:0:12193:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 02000000:00000001:2.0:1773941468.979450:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941468.979458:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941468.979461:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941468.979464:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941468.979466:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0735c00. 00000100:00080000:0.0:1773941468.979471:0:12193:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 02000000:00000001:2.0:1773941468.979473:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941468.979475:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941468.979477:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63b80. 00000100:00000001:2.0:1773941468.979480:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941468.979483:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941468.979485:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941468.979487:0:12193:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941468.979491:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941468.979492:0:12193:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941468.979495:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:0.0:1773941468.979497:0:12193:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941468.979501:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000001:00000001:0.0:1773941468.979503:0:12193:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941468.979505:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941468.979509:0:12193:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941468.979514:0:12193:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941468.979518:0:12193:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941468.979520:0:12193:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941468.979522:0:12193:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941468.979524:0:12193:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941468.979527:0:12193:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941468.979533:0:12193:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941468.979536:0:12193:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941468.979539:0:12193:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941468.979541:0:12193:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941468.979544:0:12193:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941468.979546:0:12193:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941468.979550:0:12193:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941468.979552:0:12193:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941468.979571:0:12193:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941468.979573:0:12193:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941468.979575:0:12193:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941468.979580:0:12193:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941468.979586:0:12193:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941468.979589:0:12193:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941468.979591:0:12193:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941468.979602:0:12193:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941468.979604:0:12193:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941468.983742:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec500. 00010000:00000010:1.1:1773941468.983751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:0.0:1773941469.023862:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941469.023868:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941469.023872:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941470.047874:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941470.047882:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941470.047887:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941470.239841:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941470.239848:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941470.239853:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941470.239855:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941470.239856:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941470.239857:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941470.239859:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941470.239861:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941470.239863:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941470.239866:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941470.239867:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941470.239877:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60a80. 00000020:00000040:1.0:1773941470.239882:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941470.239886:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941470.239889:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941470.239894:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941470.239900:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941470.239906:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941470.239909:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941470.239911:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941470.239918:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec500. 00010000:00000001:1.0:1773941470.239923:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941470.239926:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941470.239929:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941470.239932:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1d2b to hash 00000020:00000001:1.0:1773941470.239933:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941470.239934:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941470.239937:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941470.239940:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941470.239957:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941470.239961:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d2b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941470.239969:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d2b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941470.239982:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d2b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941470.239989:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941470.239991:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941470.239997:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941470.240000:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941470.240003:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941470.240004:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121072256 : -128249588479360 : ffff8b5b90d60a80) 00000100:00000001:1.0:1773941470.240010:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941470.240015:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941470.240017:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941470.240019:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941470.240030:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60a80 x1860112241154432/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941470.240041:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941470.240043:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941470.240047:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941470.240050:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941470.240052:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941470.240057:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241154432:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941470.240060:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941470.240065:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941470.240067:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941470.240069:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941470.240073:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734c00. 02000000:00000001:1.0:1773941470.240079:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941470.240085:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941470.240088:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941470.240093:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941470.240099:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241154432, portal 25 00000100:00000001:1.0:1773941470.240101:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941470.240102:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121072256 : -128249588479360 : ffff8b5b90d60a80) 00000100:00000040:1.0:1773941470.240106:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60a80 x1860112241154432/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941486 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941470.240112:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941470.240114:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941470.240116:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f14c8. 00000100:00000200:1.0:1773941470.240118:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241154432, offset 0 00000400:00000200:1.0:1773941470.240123:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941470.240129:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941470.240136:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941470.240140:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941470.240143:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941470.240146:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941470.240148:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941470.240151:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[80, 79] 00000400:00000200:1.0:1773941470.240153:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941470.240156:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [186:186:256:4294967295] 192.168.204.153@tcp LPNI seq info [81:81:8:4294967295] 00000400:00000200:1.0:1773941470.240159:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941470.240167:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941470.240172:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941470.240176:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b000. 00000800:00000200:1.0:1773941470.240181:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941470.240187:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941470.240191:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b000 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941470.240210:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941470.240217:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941470.240219:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294910603 00000100:00000001:1.0:1773941470.240225:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294909340 00000100:00000001:1.0:1773941470.240229:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294910603 00000100:00000001:1.0:1773941470.240231:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294909340 00000100:00000001:1.0:1773941470.240234:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941470.240236:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941470.240238:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941470.240240:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941470.240242:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941470.240244:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941470.240246:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941470.240387:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941470.240392:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b000. 00000400:00000200:3.0:1773941470.240397:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941470.240402:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941470.240407:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f14c8 00000400:00000010:3.0:1773941470.240409:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f14c8. 00000100:00000001:3.0:1773941470.240413:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941470.240417:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60a80 x1860112241154432/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941486 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941470.240425:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941470.240427:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d60a80 x1860112241154432/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941486 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941470.240442:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941470.240444:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941470.241034:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941470.241060:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941470.241063:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941470.241066:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941470.241073:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941470.241121:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941470.241129:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941470.241133:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941470.241137:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941470.241142:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605980 00000400:00000200:3.0:1773941470.241146:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941470.241152:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xe95 [1] + 224 00000800:00000001:3.0:1773941470.241160:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941470.241168:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941470.241170:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941470.241174:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941470.241179:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941470.241185:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60a80 x1860112241154432/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941486 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941470.241197:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d60a80 x1860112241154432/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941486 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941470.241215:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941470.241223:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941470.241226:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941470.241244:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941470.241249:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941470.241250:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941470.241252:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941470.241256:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941470.241261:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941470.241263:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941470.241267:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60a80 x1860112241154432/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941486 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941470.241275:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60a80 x1860112241154432/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941486 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941470.241280:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941470.241281:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941470.241283:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941470.241285:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941470.241288:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941470.241289:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941470.241291:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941470.241296:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=49 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941470.241298:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941470.241300:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941470.241303:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941470.241310:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941470.241313:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941470.241315:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941470.241318:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941470.241321:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941470.241324:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941470.241326:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941470.241327:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941470.241328:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941470.241345:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d60a80 x1860112241154432/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941486 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941470.241350:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941470.241352:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941470.241353:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941470.241355:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941470.241357:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941470.241358:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941470.241359:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941470.241360:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941470.241361:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941470.241362:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941470.241363:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941470.241365:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941470.241366:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941470.241368:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941470.241370:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 239 00000400:00000200:1.0:1773941470.241373:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941470.241376:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941470.241378:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941470.241380:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941470.241382:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941470.241384:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941470.241392:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941470.241393:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941470.241395:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941470.241397:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941470.241398:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941470.241400:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941470.241402:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941470.241403:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941470.241405:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941470.241406:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941470.241408:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941470.241410:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941470.241412:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941470.241414:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941470.241416:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941470.241417:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941470.241419:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941470.241420:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941470.241423:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60700. 00000020:00000040:1.0:1773941470.241424:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941470.241426:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941470.241427:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941470.241429:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941470.241430:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941470.241432:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941470.241435:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0736c00. 00000100:00000001:1.0:1773941470.241438:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941470.241442:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60700 x1860112241154560/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941470.241447:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941470.241449:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941470.241461:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006bd2135a] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60700 x1860112241154560/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941470.241483:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941470.241486:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941470.241487:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941470.241489:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941470.241490:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941470.241492:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941470.241493:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941470.241495:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941470.241496:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941470.241500:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60a80 x1860112241154432/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941486 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941470.241505:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941470.241507:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941470.241508:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941470.241510:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941470.241514:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60a80 x1860112241154432/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941486 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941470.241519:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241154432:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941470.241523:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941470.241524:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941470.241526:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941470.241527:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941470.241529:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60a80 x1860112241154432/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941486 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941470.241533:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941470.241534:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941470.241537:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:2.0:1773941470.241540:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941470.241541:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941470.241544:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941470.241545:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00010000:00000001:1.0:1773941470.241547:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941470.241555:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941470.241558:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941470.241559:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 00000020:00000001:1.0:1773941470.241562:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000040:2.0:1773941470.241564:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60700 x1860112241154560/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941470.241565:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00010000:1.0:1773941470.241571:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d2b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941470.241586:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941470.241588:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00010000:1.0:1773941470.241592:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d2b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941470.241593:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941470.241600:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:1.0:1773941470.241601:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000001:2.0:1773941470.241605:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00010000:1.0:1773941470.241605:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d2b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941470.241612:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941470.241614:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941470.241617:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00100000:2.0:1773941470.241621:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241154560:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00010000:1.0:1773941470.241621:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d2b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941470.241632:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941470.241633:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941470.241636:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941470.241637:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:1.0:1773941470.241639:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 02000000:00000001:2.0:1773941470.241641:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941470.241642:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d2b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941470.241645:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00010000:1.0:1773941470.241650:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 02000000:00000010:2.0:1773941470.241653:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce4800. 10000000:00000001:1.0:1773941470.241653:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941470.241656:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d2b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941470.241667:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 02000000:00000001:2.0:1773941470.241669:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941470.241670:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941470.241672:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941470.241675:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000400:00000200:2.0:1773941470.241680:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:1.0:1773941470.241680:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941470.241685:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000400:00000010:2.0:1773941470.241687:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00010000:00000001:1.0:1773941470.241688:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941470.241691:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d2b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941470.241696:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 10000000:00000001:1.0:1773941470.241696:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941470.241699:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d2b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941470.241723:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941470.241725:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941470.241726:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941470.241728:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941470.241729:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941470.241732:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941470.241733:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941470.241735:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941470.241737:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941470.241739:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1d2b from hash 00010000:00000001:1.0:1773941470.241741:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941470.241742:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941470.241743:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941470.241744:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941470.241745:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941470.241746:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941470.241748:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941470.241749:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941470.241750:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941470.241751:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241154560, portal 25 00000100:00000001:2.0:1773941470.241758:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 10000000:00000001:1.0:1773941470.241758:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941470.241763:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121071360 : -128249588480256 : ffff8b5b90d60700) 00010000:00010000:1.0:1773941470.241764:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d2b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941470.241771:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941470.241774:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000040:2.0:1773941470.241776:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60700 x1860112241154560/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941480 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941470.241776:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941470.241779:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941470.241781:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941470.241783:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941470.241785:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941470.241788:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d2b lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941470.241795:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000040:1.0:1773941470.241798:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec500 count: 0 00000100:00000040:2.0:1773941470.241802:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941470.241810:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941470.241812:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941470.241814:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941470.241819:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60a80 x1860112241154432/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941486 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941470.241824:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000400:00000010:2.0:1773941470.241825:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f345d8. 02000000:00000001:1.0:1773941470.241826:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941470.241830:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734c00. 00000100:00000200:2.0:1773941470.241832:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241154560, offset 0 02000000:00000001:1.0:1773941470.241833:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941470.241836:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941470.241838:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941470.241840:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941470.241842:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 00000400:00000200:2.0:1773941470.241844:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 02000000:00000001:1.0:1773941470.241846:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941470.241849:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941470.241850:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60a80. 00000400:00000200:2.0:1773941470.241853:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941470.241854:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941470.241856:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941470.241859:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941470.241861:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000400:00000200:2.0:1773941470.241863:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 10000000:00000001:1.0:1773941470.241864:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000400:00000200:2.0:1773941470.241869:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00000010:1.0:1773941470.241872:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0e80. 00000400:00000200:2.0:1773941470.241875:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000020:00000010:1.0:1773941470.241878:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000400:00000200:2.0:1773941470.241881:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000020:00000010:1.0:1773941470.241883:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000400:00000200:2.0:1773941470.241887:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000040:00000010:1.0:1773941470.241889:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941470.241893:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941470.241897:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 00000400:00000200:2.0:1773941470.241902:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[80, 81] 80000000:00000010:1.0:1773941470.241903:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941470.241909:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 00000400:00000200:2.0:1773941470.241912:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 20000000:00000010:1.0:1773941470.241914:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 00000400:00000200:2.0:1773941470.241920:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [187:187:256:4294967295] 192.168.204.143@tcp LPNI seq info [82:82:8:4294967295] 10000000:00000040:1.0:1773941470.241921:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941470.241925:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941470.241932:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000400:00000200:2.0:1773941470.241933:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000040:00000010:1.0:1773941470.241936:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941470.241939:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941470.241944:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941470.241948:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00000400:00000200:2.0:1773941470.241949:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00040000:00000010:1.0:1773941470.241952:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941470.241956:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941470.241960:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 00000800:00000200:2.0:1773941470.241961:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 10000000:00000010:1.0:1773941470.241963:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0e80. 10000000:01000000:1.0:1773941470.241967:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000800:00000010:2.0:1773941470.241968:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0200. 10000000:00000001:1.0:1773941470.241969:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941470.241974:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941470.241978:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 00000800:00000200:2.0:1773941470.241979:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00000001:1.0:1773941470.241981:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941470.241985:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941470.241988:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000200:2.0:1773941470.241991:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941470.241996:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0200 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941470.242011:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941470.242015:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941470.242018:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294910605 00000100:00000001:2.0:1773941470.242022:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294910603 00000100:00000001:2.0:1773941470.242026:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294910605 00000100:00000001:2.0:1773941470.242030:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294910603 00000100:00000001:2.0:1773941470.242034:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941470.242038:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941470.242041:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941470.242044:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941470.242049:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941470.242051:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941470.242054:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941470.242056:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941470.242058:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941470.242061:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941470.242087:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941470.242091:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0200. 00000400:00000200:3.0:1773941470.242097:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941470.242102:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941470.242105:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f345d8 00000400:00000010:3.0:1773941470.242113:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f345d8. 00000100:00000001:3.0:1773941470.242116:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941470.242121:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60700 x1860112241154560/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941480 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941470.242131:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941470.242178:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60700 x1860112241154560/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941480 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941470.242187:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941470.242189:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941470.243388:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941470.243396:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941470.243399:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941470.243402:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941470.243407:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941470.243413:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941470.243420:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941470.243423:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941470.243427:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941470.243431:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605a00 00000400:00000200:3.0:1773941470.243436:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941470.243487:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xe9d [1] + 0 00000800:00000001:3.0:1773941470.243493:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941470.243500:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941470.243503:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941470.243507:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941470.243511:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941470.243516:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60700 x1860112241154560/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941480 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941470.243528:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60700 x1860112241154560/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941480 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941470.243547:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941470.243561:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941470.243565:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941470.243576:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941470.243579:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941470.243583:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941470.243585:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941470.243586:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941470.243590:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941470.243597:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941470.243599:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941470.243603:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60700 x1860112241154560/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941480 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941470.243618:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60700 x1860112241154560/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941480 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941470.243623:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941470.243624:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941470.243626:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941470.243628:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941470.243629:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941470.243631:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941470.243633:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941470.243636:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=49 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941470.243639:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941470.243641:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941470.243642:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941470.243644:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941470.243646:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941470.243647:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941470.243651:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60700 x1860112241154560/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941480 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941470.243655:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941470.243657:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941470.243659:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941470.243661:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941470.243665:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941470.243667:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941470.243668:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941470.243670:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941470.243672:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941470.243674:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941470.243675:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941470.243678:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941470 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941470.243688:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941470.243695:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941470.243698:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941470.243700:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941470.243724:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941470.243727:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941470.243733:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941470.243736:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941470.243738:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941470.243743:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941470.243747:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941470.243750:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941470.243973:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941470.243976:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941470.243981:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60700 x1860112241154560/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941480 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941470.243988:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241154560:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941470.243990:0:12194:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941470.243993:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941470.243996:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941470.243998:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60700 x1860112241154560/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941480 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941470.244003:0:12194:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:2.0:1773941470.244007:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941470.244011:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:1.0:1773941470.244011:0:12194:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000010:2.0:1773941470.244014:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce4800. 00000100:00000001:1.0:1773941470.244016:0:12194:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941470.244018:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:01000000:1.0:1773941470.244020:0:12194:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000001:2.0:1773941470.244021:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941470.244024:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:1.0:1773941470.244026:0:12194:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941470.244027:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941470.244031:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0736c00. 00000001:00000001:1.0:1773941470.244033:0:12194:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941470.244034:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941470.244035:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941470.244036:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60700. 00000001:00000001:1.0:1773941470.244045:0:12194:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941470.244047:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941470.244050:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941470.244051:0:12194:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941470.244053:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941470.244055:0:12194:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941470.244058:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000001:00000001:1.0:1773941470.244059:0:12194:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941470.244061:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941470.244064:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000001:1.0:1773941470.244064:0:12194:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941470.244067:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941470.244069:0:12194:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941470.244076:0:12194:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941470.244079:0:12194:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941470.244083:0:12194:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941470.244084:0:12194:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941470.244086:0:12194:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941470.244088:0:12194:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941470.244089:0:12194:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941470.244090:0:12194:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941470.244119:0:12194:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941470.244121:0:12194:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941470.244123:0:12194:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941470.244127:0:12194:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941470.244130:0:12194:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941470.244132:0:12194:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941470.244133:0:12194:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941470.244135:0:12194:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941470.244137:0:12194:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941470.247738:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec500. 00010000:00000010:1.1:1773941470.247747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:0.0:1773941471.071829:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941471.071834:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941471.071839:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941471.583808:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941471.583815:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941471.583818:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941471.583820:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941471.583822:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941471.583823:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941471.583825:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941471.583827:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941471.583828:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941471.583831:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941471.583832:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941471.583843:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90274a80. 00000020:00000040:3.0:1773941471.583847:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941471.583850:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941471.583853:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941471.583857:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941471.583863:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e3a00. 00000100:00000001:3.0:1773941471.583869:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941471.583873:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941471.583875:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941471.583881:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962e00. 00010000:00000001:3.0:1773941471.583886:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941471.583889:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485400. 00000020:00000001:3.0:1773941471.583892:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941471.583895:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485400 with handle 0xa33a5be14eae1d32 to hash 00000020:00000001:3.0:1773941471.583896:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941471.583897:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860126720 : -128249849424896 : ffff8b5b81485400) 00010000:00000001:3.0:1773941471.583899:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860126720 : -128249849424896 : ffff8b5b81485400) 00010000:00000001:3.0:1773941471.583903:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941471.583905:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941471.583909:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1d32 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941471.583916:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1d32 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941471.583937:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1d32 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941471.583942:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941471.583944:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941471.583949:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941471.583950:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941471.583953:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941471.583954:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109619840 : -128249599931776 : ffff8b5b90274a80) 00000100:00000001:3.0:1773941471.583958:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941471.583962:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941471.583964:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941471.583965:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941471.583973:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90274a80 x1860112241154688/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941471.583985:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941471.583987:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941471.583990:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941471.583992:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941471.583993:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941471.583996:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90274a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241154688:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941471.584000:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941471.584004:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941471.584006:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941471.584008:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941471.584012:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941471.584013:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941471.584020:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941471.584022:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab9a0. 00000400:00000010:3.0:1773941471.584027:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a908. 00000100:00000200:3.0:1773941471.584032:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241154688, portal 25 00000100:00000001:3.0:1773941471.584034:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941471.584038:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109619840 : -128249599931776 : ffff8b5b90274a80) 00000100:00000040:3.0:1773941471.584042:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90274a80 x1860112241154688/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941487 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941471.584047:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941471.584049:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941471.584051:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a2a8. 00000100:00000200:3.0:1773941471.584054:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241154688, offset 0 00000400:00000200:3.0:1773941471.584058:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941471.584064:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941471.584070:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941471.584074:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941471.584077:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941471.584081:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941471.584083:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941471.584085:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[82, 81] 00000400:00000200:3.0:1773941471.584088:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941471.584091:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [188:188:256:4294967295] 192.168.204.153@tcp LPNI seq info [83:83:8:4294967295] 00000400:00000200:3.0:1773941471.584094:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941471.584102:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941471.584108:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941471.584113:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69600. 00000800:00000200:3.0:1773941471.584120:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941471.584128:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941471.584133:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941471.584154:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941471.584159:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941471.584162:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294911947 00000100:00000001:3.0:1773941471.584168:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294910605 00000100:00000001:3.0:1773941471.584172:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294911947 00000100:00000001:3.0:1773941471.584174:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294910605 00000100:00000001:3.0:1773941471.584181:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941471.584183:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941471.584185:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941471.584187:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941471.584189:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941471.584191:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941471.584194:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941471.584281:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941471.584285:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69600. 00000400:00000200:2.0:1773941471.584290:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941471.584295:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941471.584299:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a2a8 00000400:00000010:2.0:1773941471.584301:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a2a8. 00000100:00000001:2.0:1773941471.584322:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941471.584330:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90274a80 x1860112241154688/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941487 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941471.584338:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941471.584341:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90274a80 x1860112241154688/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941487 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941471.584346:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941471.584348:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941471.584939:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941471.584948:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941471.584950:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941471.584952:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941471.584956:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941471.584959:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941471.584963:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941471.584970:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941471.584973:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941471.584976:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605a80 00000400:00000200:3.0:1773941471.584979:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941471.585032:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xead [1] + 224 00000800:00000001:3.0:1773941471.585038:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941471.585044:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941471.585045:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941471.585048:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941471.585051:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941471.585055:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90274a80 x1860112241154688/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941487 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941471.585062:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90274a80 x1860112241154688/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941487 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941471.585076:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941471.585081:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941471.585083:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941471.585151:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941471.585157:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941471.585158:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941471.585161:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab9a0. 00000400:00000200:1.0:1773941471.585165:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a908 00000400:00000010:1.0:1773941471.585166:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a908. 00000100:00000001:1.0:1773941471.585169:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941471.585173:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90274a80 x1860112241154688/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941487 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941471.585181:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90274a80 x1860112241154688/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941487 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941471.585187:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941471.585195:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941471.585197:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941471.585199:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941471.585201:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941471.585203:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941471.585205:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941471.585209:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=50 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941471.585212:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941471.585214:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941471.585218:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941471.585220:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941471.585222:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941471.585225:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941471.585228:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941471.585231:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941471.585234:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941471.585235:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941471.585236:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941471.585238:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941471.585240:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90274a80 x1860112241154688/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941487 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941471.585245:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941471.585246:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941471.585247:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941471.585249:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941471.585251:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941471.585252:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941471.585254:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941471.585254:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941471.585255:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941471.585256:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941471.585257:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941471.585259:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941471.585260:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941471.585265:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941471.585267:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 240 00000400:00000200:1.0:1773941471.585271:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941471.585274:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941471.585277:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941471.585280:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941471.585281:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941471.585283:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941471.585285:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941471.585287:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941471.585288:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941471.585290:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941471.585291:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941471.585293:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941471.585295:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941471.585296:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941471.585297:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941471.585299:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941471.585301:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941471.585303:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941471.585318:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941471.585320:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941471.585322:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941471.585324:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941471.585326:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941471.585327:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941471.585331:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60a80. 00000020:00000040:1.0:1773941471.585333:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941471.585336:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941471.585337:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941471.585338:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941471.585340:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941471.585345:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941471.585349:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734c00. 00000100:00000001:1.0:1773941471.585352:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941471.585357:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60a80 x1860112241154816/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941471.585363:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941471.585365:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941471.585368:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000079bd9b68] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60a80 x1860112241154816/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941471.585383:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941471.585385:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941471.585387:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941471.585389:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941471.585391:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941471.585392:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941471.585393:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941471.585395:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941471.585397:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941471.585400:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90274a80 x1860112241154688/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941487 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941471.585406:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941471.585408:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941471.585409:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941471.585410:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941471.585413:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941471.585413:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941471.585416:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941471.585420:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90274a80 x1860112241154688/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941487 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941471.585425:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60a80 x1860112241154816/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941471.585444:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1773941471.585444:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90274a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241154688:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941471.585446:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941471.585451:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941471.585453:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941471.585454:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941471.585458:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241154816:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941471.585460:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941471.585463:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941471.585464:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941471.585466:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:2.0:1773941471.585472:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941471.585475:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941471.585476:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941471.585479:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1773941471.585483:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90274a80 x1860112241154688/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941487 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1773941471.585484:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce7800. 02000000:00000001:2.0:1773941471.585490:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941471.585496:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941471.585498:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941471.585500:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000400:00000010:2.0:1773941471.585502:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941471.585506:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000001:1.0:1773941471.585508:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000200:2.0:1773941471.585511:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241154816, portal 25 00000100:00000001:1.0:1773941471.585511:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941471.585516:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941471.585519:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121072256 : -128249588479360 : ffff8b5b90d60a80) 00010000:00000001:1.0:1773941471.585519:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941471.585524:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000040:2.0:1773941471.585527:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60a80 x1860112241154816/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941481 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1773941471.585528:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941471.585532:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485400 refcount=4 00000100:00000001:2.0:1773941471.585538:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1773941471.585539:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860126720 : -128249849424896 : ffff8b5b81485400) 00000100:00000040:2.0:1773941471.585542:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941471.585547:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860126720 : -128249849424896 : ffff8b5b81485400) 00000400:00000010:2.0:1773941471.585548:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34000. 00000100:00000200:2.0:1773941471.585552:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241154816, offset 0 00000400:00000200:2.0:1773941471.585557:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941471.585557:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1d32 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941471.585565:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941471.585570:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941471.585575:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941471.585577:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:2.0:1773941471.585579:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941471.585585:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941471.585588:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:1.0:1773941471.585588:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1d32 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941471.585595:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[82, 83] 00000400:00000200:2.0:1773941471.585605:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941471.585609:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:2.0:1773941471.585610:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [189:189:256:4294967295] 192.168.204.143@tcp LPNI seq info [84:84:8:4294967295] 00010000:00010000:1.0:1773941471.585615:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1d32 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941471.585616:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941471.585628:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941471.585628:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941471.585632:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941471.585635:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1d32 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941471.585636:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941471.585642:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0800. 00010000:00000001:1.0:1773941471.585646:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000800:00000200:2.0:1773941471.585650:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941471.585650:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941471.585653:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000800:00000200:2.0:1773941471.585663:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00010000:1.0:1773941471.585664:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1d32 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941471.585667:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0800 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:1.0:1773941471.585676:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485400 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941471.585678:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941471.585680:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1d32 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941471.585684:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941471.585685:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941471.585688:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941471.585690:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941471.585691:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000040:1.0:1773941471.585692:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485400 refcount=5 00000100:00000001:2.0:1773941471.585696:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294911948 00000020:00000001:1.0:1773941471.585697:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860126720 : -128249849424896 : ffff8b5b81485400) 00010000:00000001:1.0:1773941471.585736:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860126720 : -128249849424896 : ffff8b5b81485400) 00010000:00000001:1.0:1773941471.585740:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941471.585742:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1d32 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941471.585743:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294911947 00000100:00000001:2.0:1773941471.585751:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294911949 10000000:00000001:1.0:1773941471.585752:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941471.585756:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1d32 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941471.585757:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941471.585763:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0800. 10000000:01000000:1.0:1773941471.585765:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941471.585768:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941471.585769:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941471.585772:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294911947 00010000:00010000:1.0:1773941471.585772:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941471.585775:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:3.0:1773941471.585777:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941471.585778:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941471.585779:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941471.585783:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:2.0:1773941471.585784:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941471.585787:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941471.585789:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941471.585789:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941471.585791:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941471.585795:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1773941471.585797:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485400 with handle 0xa33a5be14eae1d32 from hash 00000400:00000200:3.0:1773941471.585798:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34000 00010000:00000001:1.0:1773941471.585801:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000010:3.0:1773941471.585802:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34000. 00000100:00000001:2.0:1773941471.585803:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941471.585808:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941471.585811:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941471.585813:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000200:3.0:1773941471.585815:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60a80 x1860112241154816/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941481 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941471.585815:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941471.585817:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1773941471.585818:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:2.0:1773941471.585821:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941471.585823:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:2.0:1773941471.585825:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941471.585827:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:2.0:1773941471.585828:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941471.585831:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941471.585833:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941471.585835:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941471.585836:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941471.585839:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1d32 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941471.585843:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941471.585844:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941471.585845:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941471.585846:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941471.585847:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941471.585848:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941471.585848:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941471.585850:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485400/0xa33a5be14eae1d32 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941471.585854:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962e00 count: 0 00010000:00000001:1.0:1773941471.585862:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941471.585863:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941471.585865:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941471.585869:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90274a80 x1860112241154688/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941487 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941471.585874:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941471.585876:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941471.585878:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 02000000:00000001:1.0:1773941471.585880:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941471.585882:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941471.585883:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941471.585888:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941471.585892:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e3a00. 00000100:00000001:3.0:1773941471.585894:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941471.585900:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60a80 x1860112241154816/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941481 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941471.585901:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941471.585904:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941471.585907:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90274a80. 00000100:00000001:3.0:1773941471.585913:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941471.585913:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941471.585916:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941471.585917:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000001:1.0:1773941471.585920:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941471.585923:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941471.585925:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941471.585928:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0e80. 00000020:00000010:1.0:1773941471.585931:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941471.585935:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941471.585938:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941471.585941:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941471.585944:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941471.585948:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941471.585951:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941471.585955:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941471.585958:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941471.585960:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941471.585963:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941471.585966:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941471.585968:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941471.585971:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941471.585973:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941471.585977:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941471.585978:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941471.585981:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941471.585982:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0e80. 10000000:01000000:1.0:1773941471.585984:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941471.585986:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941471.585988:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941471.585989:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941471.585990:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941471.585991:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941471.585996:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941471.587221:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941471.587227:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941471.587229:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941471.587231:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941471.587234:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941471.587238:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941471.587241:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941471.587244:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941471.587277:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941471.587280:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605b00 00000400:00000200:3.0:1773941471.587283:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941471.587286:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xebd [1] + 0 00000800:00000001:3.0:1773941471.587290:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941471.587295:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941471.587297:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941471.587299:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941471.587302:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941471.587322:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60a80 x1860112241154816/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941481 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941471.587333:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60a80 x1860112241154816/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941481 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941471.587354:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941471.587359:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941471.587362:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941471.587388:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941471.587391:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941471.587394:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941471.587395:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941471.587397:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941471.587400:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941471.587402:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941471.587404:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941471.587409:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60a80 x1860112241154816/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941481 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941471.587416:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60a80 x1860112241154816/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941481 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941471.587422:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941471.587423:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941471.587425:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941471.587431:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941471.587432:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941471.587434:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941471.587435:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941471.587438:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=50 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941471.587441:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941471.587442:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941471.587444:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941471.587445:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941471.587447:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941471.587449:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941471.587452:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60a80 x1860112241154816/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941481 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941471.587457:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941471.587459:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941471.587460:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941471.587462:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941471.587465:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941471.587466:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941471.587467:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941471.587468:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941471.587470:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941471.587471:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941471.587473:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941471.587476:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941471 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941471.587478:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941471.587490:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941471.587492:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941471.587493:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941471.587494:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941471.587495:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941471.587498:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941471.587500:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941471.587501:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941471.587508:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941471.587511:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941471.587513:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941471.587730:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941471.587734:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941471.587738:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60a80 x1860112241154816/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941481 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941471.587743:0:12195:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941471.587746:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241154816:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941471.587751:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941471.587753:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1773941471.587758:0:12195:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000040:2.0:1773941471.587759:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60a80 x1860112241154816/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941481 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941471.587766:0:12195:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941471.587768:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941471.587771:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1773941471.587773:0:12195:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:2.0:1773941471.587774:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce7800. 10000000:01000000:1.0:1773941471.587778:0:12195:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941471.587779:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941471.587782:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941471.587784:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:1.0:1773941471.587785:0:12195:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941471.587786:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941471.587789:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734c00. 00000001:00000001:1.0:1773941471.587791:0:12195:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941471.587792:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941471.587795:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941471.587797:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60a80. 00000001:00000001:1.0:1773941471.587798:0:12195:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941471.587799:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941471.587802:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941471.587805:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941471.587805:0:12195:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941471.587810:0:12195:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941471.587812:0:12195:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941471.587817:0:12195:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941471.587818:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941471.587819:0:12195:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941471.587821:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941471.587824:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941471.587826:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941471.587827:0:12195:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941471.587832:0:12195:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941471.587834:0:12195:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941471.587837:0:12195:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941471.587840:0:12195:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941471.587843:0:12195:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941471.587844:0:12195:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941471.587846:0:12195:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941471.587863:0:12195:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941471.587865:0:12195:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941471.587868:0:12195:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941471.587873:0:12195:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941471.587877:0:12195:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941471.587881:0:12195:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941471.587883:0:12195:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941471.587886:0:12195:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941471.587888:0:12195:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941471.591747:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962e00. 00010000:00000010:1.1:1773941471.591757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485400. 00000020:00000001:3.0:1773941472.095907:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941472.095914:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941472.095919:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941472.863754:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941472.863760:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941472.863764:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941472.863766:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941472.863768:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941472.863769:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941472.863771:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941472.863773:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941472.863775:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941472.863777:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941472.863778:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941472.863787:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90274a80. 00000020:00000040:3.0:1773941472.863791:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941472.863795:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941472.863798:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941472.863802:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941472.863810:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e3000. 00000100:00000001:3.0:1773941472.863816:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941472.863820:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941472.863822:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941472.863828:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962d00. 00010000:00000001:3.0:1773941472.863833:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941472.863836:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81484c00. 00000020:00000001:3.0:1773941472.863840:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941472.863842:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81484c00 with handle 0xa33a5be14eae1d39 to hash 00000020:00000001:3.0:1773941472.863843:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941472.863844:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860124672 : -128249849426944 : ffff8b5b81484c00) 00010000:00000001:3.0:1773941472.863847:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860124672 : -128249849426944 : ffff8b5b81484c00) 00010000:00000001:3.0:1773941472.863863:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941472.863866:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941472.863869:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1d39 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941472.863877:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1d39 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941472.863887:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1d39 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941472.863891:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941472.863892:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941472.863897:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e300. 00000100:00000001:3.0:1773941472.863899:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387400448 : -128249322151168 : ffff8b5ba0b5e300) 00000100:00000001:3.0:1773941472.863901:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941472.863902:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109619840 : -128249599931776 : ffff8b5b90274a80) 00000100:00000001:3.0:1773941472.863906:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941472.863909:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941472.863911:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941472.863913:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941472.863920:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90274a80 x1860112241154944/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941472.863927:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941472.863929:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941472.863932:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941472.863934:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941472.863935:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941472.863938:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90274a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241154944:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941472.863941:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941472.863945:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941472.863947:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941472.863949:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941472.863958:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90262000. 02000000:00000001:3.0:1773941472.863959:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941472.863965:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941472.863968:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab7e8. 00000400:00000010:3.0:1773941472.863973:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6ac38. 00000100:00000200:3.0:1773941472.863978:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241154944, portal 25 00000100:00000001:3.0:1773941472.863979:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941472.863980:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109619840 : -128249599931776 : ffff8b5b90274a80) 00000100:00000040:3.0:1773941472.863984:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90274a80 x1860112241154944/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941488 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941472.863989:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941472.863992:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941472.863993:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a5d8. 00000100:00000200:3.0:1773941472.863995:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241154944, offset 0 00000400:00000200:3.0:1773941472.864000:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941472.864006:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941472.864012:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941472.864016:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941472.864019:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941472.864023:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941472.864025:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941472.864027:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[84, 83] 00000400:00000200:3.0:1773941472.864030:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941472.864033:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [190:190:256:4294967295] 192.168.204.153@tcp LPNI seq info [85:85:8:4294967295] 00000400:00000200:3.0:1773941472.864036:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941472.864043:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941472.864047:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941472.864051:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69600. 00000800:00000200:3.0:1773941472.864056:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941472.864062:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941472.864065:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941472.864086:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941472.864090:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941472.864092:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294913227 00000100:00000001:3.0:1773941472.864098:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294911948 00000100:00000001:3.0:1773941472.864102:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294913227 00000100:00000001:3.0:1773941472.864106:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294911949 00000100:00000001:3.0:1773941472.864110:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941472.864113:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941472.864115:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941472.864118:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941472.864121:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 11 seconds 00000100:00000001:3.0:1773941472.864125:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941472.864128:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941472.864217:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941472.864222:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69600. 00000400:00000200:2.0:1773941472.864227:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941472.864237:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941472.864242:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a5d8 00000400:00000010:2.0:1773941472.864244:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a5d8. 00000100:00000001:2.0:1773941472.864248:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941472.864252:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90274a80 x1860112241154944/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941488 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941472.864260:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941472.864263:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90274a80 x1860112241154944/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941488 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941472.864268:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941472.864270:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941472.865009:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941472.865018:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941472.865021:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941472.865023:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941472.865027:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941472.865031:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941472.865035:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941472.865070:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941472.865073:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941472.865076:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605b80 00000400:00000200:3.0:1773941472.865079:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941472.865083:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xecd [1] + 224 00000800:00000001:3.0:1773941472.865088:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941472.865093:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941472.865095:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941472.865097:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941472.865101:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941472.865106:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90274a80 x1860112241154944/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941488 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941472.865132:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90274a80 x1860112241154944/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941488 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941472.865150:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941472.865158:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941472.865161:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941472.865191:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941472.865198:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941472.865199:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941472.865201:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab7e8. 00000400:00000200:1.0:1773941472.865205:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6ac38 00000400:00000010:1.0:1773941472.865213:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6ac38. 00000100:00000001:1.0:1773941472.865216:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941472.865221:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90274a80 x1860112241154944/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941488 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941472.865228:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90274a80 x1860112241154944/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941488 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941472.865234:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941472.865235:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941472.865237:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941472.865239:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941472.865241:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941472.865242:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941472.865243:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941472.865248:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=51 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941472.865250:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941472.865253:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941472.865256:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941472.865258:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941472.865260:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941472.865263:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941472.865266:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941472.865269:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941472.865272:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941472.865273:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941472.865274:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941472.865276:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941472.865278:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90274a80 x1860112241154944/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941488 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941472.865296:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941472.865297:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941472.865298:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941472.865305:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941472.865307:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941472.865308:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941472.865310:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941472.865311:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941472.865311:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941472.865312:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941472.865314:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941472.865316:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941472.865317:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941472.865320:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941472.865321:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 242 00000400:00000200:1.0:1773941472.865325:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941472.865328:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941472.865331:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941472.865333:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941472.865335:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941472.865336:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941472.865338:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941472.865340:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941472.865341:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941472.865343:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941472.865344:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941472.865346:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941472.865348:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941472.865349:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941472.865350:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941472.865351:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941472.865353:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941472.865355:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941472.865357:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941472.865359:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941472.865361:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941472.865365:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941472.865367:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941472.865369:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941472.865372:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60380. 00000020:00000040:1.0:1773941472.865374:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941472.865376:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941472.865377:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941472.865379:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941472.865380:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941472.865382:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941472.865385:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737400. 00000100:00000001:1.0:1773941472.865388:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941472.865392:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60380 x1860112241155072/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941472.865398:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941472.865400:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941472.865403:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077e2ee70] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60380 x1860112241155072/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941472.865419:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941472.865421:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941472.865422:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941472.865424:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941472.865426:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941472.865427:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941472.865428:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 1 seconds 00000100:00000001:1.0:1773941472.865430:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941472.865432:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941472.865435:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90274a80 x1860112241154944/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941488 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941472.865441:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941472.865443:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941472.865445:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941472.865446:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941472.865452:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941472.865456:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1773941472.865457:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90274a80 x1860112241154944/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941488 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941472.865458:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941472.865466:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60380 x1860112241155072/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941472.865483:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941472.865486:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941472.865487:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90274a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241154944:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941472.865488:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941472.865491:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941472.865495:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941472.865500:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241155072:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941472.865500:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941472.865504:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941472.865507:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941472.865508:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941472.865510:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941472.865513:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941472.865515:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000010:2.0:1773941472.865518:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce4400. 00000100:00000001:1.0:1773941472.865518:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941472.865522:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941472.865526:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90274a80 x1860112241154944/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941488 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941472.865529:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941472.865535:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000100:00000001:1.0:1773941472.865540:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1773941472.865542:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000010:1.0:1773941472.865543:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e300. 00000100:00000200:2.0:1773941472.865546:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241155072, portal 25 00000100:00000001:1.0:1773941472.865547:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941472.865551:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941472.865552:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941472.865555:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941472.865557:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941472.865558:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941472.865561:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121070464 : -128249588481152 : ffff8b5b90d60380) 00000020:00000040:1.0:1773941472.865561:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484c00 refcount=4 00000020:00000001:1.0:1773941472.865567:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860124672 : -128249849426944 : ffff8b5b81484c00) 00010000:00000001:1.0:1773941472.865569:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860124672 : -128249849426944 : ffff8b5b81484c00) 00000100:00000040:2.0:1773941472.865571:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60380 x1860112241155072/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941472.865574:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1d39 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941472.865593:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941472.865595:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:2.0:1773941472.865599:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941472.865602:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1d39 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941472.865603:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34110. 00000100:00000200:2.0:1773941472.865607:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241155072, offset 0 00010000:00000001:1.0:1773941472.865614:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:2.0:1773941472.865616:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941472.865618:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1d39 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941472.865624:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941472.865630:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941472.865633:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941472.865635:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941472.865639:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941472.865641:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941472.865643:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[84, 85] 00000400:00000200:2.0:1773941472.865646:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941472.865651:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [191:191:256:4294967295] 192.168.204.143@tcp LPNI seq info [86:86:8:4294967295] 00010000:00000001:1.0:1773941472.865660:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941472.865661:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941472.865664:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941472.865667:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1d39 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941472.865672:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941472.865682:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000800:00000200:2.0:1773941472.865683:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941472.865685:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1773941472.865688:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0e00. 00010000:00000001:1.0:1773941472.865689:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941472.865693:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1d39 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941472.865696:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00010000:1.0:1773941472.865736:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81484c00 already unused, calling callback (ffffffffc1482c20) 00000800:00000200:2.0:1773941472.865741:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 10000000:00000001:1.0:1773941472.865741:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941472.865745:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1d39 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941472.865748:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0e00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941472.865757:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941472.865759:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941472.865760:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941472.865762:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484c00 refcount=5 00000020:00000001:1.0:1773941472.865763:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860124672 : -128249849426944 : ffff8b5b81484c00) 00010000:00000001:1.0:1773941472.865765:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860124672 : -128249849426944 : ffff8b5b81484c00) 00010000:00000001:1.0:1773941472.865767:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000001:2.0:1773941472.865769:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941472.865770:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1d39 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941472.865780:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:1.0:1773941472.865781:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:2.0:1773941472.865786:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294913229 10000000:00010000:1.0:1773941472.865786:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1d39 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941472.865794:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294913227 10000000:01000000:1.0:1773941472.865795:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:2.0:1773941472.865799:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294913229 00000100:00000001:2.0:1773941472.865803:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294913227 10000000:00000040:1.0:1773941472.865805:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941472.865806:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941472.865810:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941472.865813:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:2.0:1773941472.865816:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941472.865817:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:2.0:1773941472.865820:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941472.865821:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:2.0:1773941472.865825:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941472.865826:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:2.0:1773941472.865829:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941472.865829:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941472.865834:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81484c00 with handle 0xa33a5be14eae1d39 from hash 00000800:00000200:3.0:1773941472.865836:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941472.865836:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:2.0:1773941472.865837:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941472.865841:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000010:3.0:1773941472.865842:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0e00. 00000100:00000001:2.0:1773941472.865842:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:1.0:1773941472.865845:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941472.865847:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941472.865849:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941472.865849:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000400:00000200:3.0:1773941472.865851:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:1.0:1773941472.865853:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:2.0:1773941472.865854:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941472.865857:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:2.0:1773941472.865858:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941472.865860:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00010000:00000001:1.0:1773941472.865862:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941472.865865:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941472.865866:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34110 00010000:00000001:1.0:1773941472.865867:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941472.865870:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34110. 10000000:00000001:1.0:1773941472.865871:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941472.865876:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1d39 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941472.865877:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941472.865884:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60380 x1860112241155072/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941472.865885:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941472.865888:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941472.865890:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941472.865892:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941472.865895:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941472.865897:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941472.865898:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941472.865899:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81484c00/0xa33a5be14eae1d39 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941472.865904:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962d00 count: 0 00010000:00000001:1.0:1773941472.865913:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941472.865914:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941472.865916:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941472.865933:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90274a80 x1860112241154944/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941488 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941472.865939:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941472.865941:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941472.865944:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90262000. 02000000:00000001:1.0:1773941472.865947:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941472.865949:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1773941472.865949:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941472.865952:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941472.865955:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000040:3.0:1773941472.865956:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60380 x1860112241155072/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941482 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1773941472.865958:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e3000. 02000000:00000001:1.0:1773941472.865963:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941472.865966:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941472.865968:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941472.865969:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90274a80. 00000100:00000001:3.0:1773941472.865971:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1773941472.865973:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941472.865975:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941472.865977:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941472.865979:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941472.865981:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941472.865983:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0e80. 00000020:00000010:1.0:1773941472.865987:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941472.865991:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941472.865994:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941472.865998:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941472.866001:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941472.866004:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941472.866008:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941472.866011:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 10000000:00000040:1.0:1773941472.866015:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941472.866016:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941472.866020:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941472.866023:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941472.866025:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941472.866029:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941472.866030:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941472.866037:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941472.866039:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941472.866042:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941472.866044:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0e80. 10000000:01000000:1.0:1773941472.866046:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941472.866048:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941472.866049:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941472.866051:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941472.866052:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941472.866053:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941472.866054:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941472.866992:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941472.866997:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941472.866999:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941472.867001:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941472.867005:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941472.867009:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941472.867014:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941472.867076:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941472.867081:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941472.867085:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605c00 00000400:00000200:3.0:1773941472.867089:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941472.867093:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xedd [1] + 0 00000800:00000001:3.0:1773941472.867106:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941472.867112:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941472.867114:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941472.867118:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941472.867122:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941472.867127:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60380 x1860112241155072/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941482 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941472.867137:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60380 x1860112241155072/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941482 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941472.867153:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941472.867158:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941472.867161:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941472.867176:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941472.867178:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941472.867181:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941472.867183:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941472.867184:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941472.867188:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941472.867189:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941472.867191:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941472.867195:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60380 x1860112241155072/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941482 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941472.867201:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60380 x1860112241155072/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941482 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941472.867206:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941472.867207:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941472.867208:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941472.867210:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941472.867211:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941472.867212:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941472.867214:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941472.867216:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=51 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941472.867218:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941472.867220:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941472.867221:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941472.867223:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941472.867225:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941472.867226:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941472.867234:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60380 x1860112241155072/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941482 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941472.867239:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941472.867241:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941472.867242:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941472.867244:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941472.867247:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941472.867248:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941472.867249:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941472.867251:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941472.867252:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941472.867254:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941472.867256:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941472.867259:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941472 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941472.867260:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941472.867265:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941472.867267:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941472.867268:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941472.867269:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941472.867270:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941472.867272:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941472.867274:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941472.867276:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941472.867278:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941472.867294:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941472.867297:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941472.867478:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941472.867481:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941472.867485:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60380 x1860112241155072/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941482 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941472.867492:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241155072:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941472.867499:0:12196:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941472.867501:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941472.867504:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1773941472.867510:0:12196:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000040:2.0:1773941472.867511:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60380 x1860112241155072/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941482 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941472.867516:0:12196:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941472.867520:0:12196:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941472.867521:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 10000000:01000000:1.0:1773941472.867523:0:12196:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941472.867525:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941472.867527:0:12196:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941472.867528:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce4400. 00000001:00000001:1.0:1773941472.867531:0:12196:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941472.867532:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941472.867535:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:1.0:1773941472.867535:0:12196:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941472.867537:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:1.0:1773941472.867539:0:12196:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941472.867540:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941472.867542:0:12196:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941472.867544:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737400. 00000001:00000001:1.0:1773941472.867545:0:12196:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941472.867548:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000001:1.0:1773941472.867549:0:12196:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941472.867551:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941472.867552:0:12196:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000010:2.0:1773941472.867554:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60380. 00000100:00080000:1.0:1773941472.867557:0:12196:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941472.867558:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941472.867561:0:12196:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941472.867562:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941472.867566:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941472.867566:0:12196:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941472.867569:0:12196:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941472.867572:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1773941472.867573:0:12196:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941472.867574:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941472.867578:0:12196:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941472.867579:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:01000000:1.0:1773941472.867581:0:12196:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:2.0:1773941472.867582:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:1.0:1773941472.867585:0:12196:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941472.867600:0:12196:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941472.867601:0:12196:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941472.867603:0:12196:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941472.867605:0:12196:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941472.867612:0:12196:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941472.867614:0:12196:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941472.867616:0:12196:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941472.867618:0:12196:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941472.867619:0:12196:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941472.871736:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962d00. 00010000:00000010:1.1:1773941472.871745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81484c00. 00000020:00000001:0.0:1773941473.119873:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941473.119879:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941473.119900:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941473.183840:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (248) 00000020:00000001:3.0:1773941474.143865:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941474.143871:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941474.143875:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941474.463876:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941474.463885:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941474.463891:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941474.463893:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941474.463895:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941474.463896:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941474.463899:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941474.463901:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941474.463904:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941474.463907:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941474.463909:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941474.463920:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60000. 00000020:00000040:1.0:1773941474.463926:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941474.463931:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941474.463934:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941474.463939:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941474.463946:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941474.463954:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941474.463959:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941474.463962:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941474.463970:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890eca00. 00010000:00000001:1.0:1773941474.463976:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941474.463980:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941474.463985:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941474.463988:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1d40 to hash 00000020:00000001:1.0:1773941474.463991:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941474.464003:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941474.464007:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941474.464011:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941474.464015:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941474.464033:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d40 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941474.464046:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d40 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941474.464059:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d40 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941474.464066:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941474.464068:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941474.464076:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941474.464078:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941474.464081:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941474.464083:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121069568 : -128249588482048 : ffff8b5b90d60000) 00000100:00000001:1.0:1773941474.464088:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941474.464093:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941474.464095:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941474.464098:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941474.464108:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60000 x1860112241155200/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941474.464135:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941474.464138:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941474.464143:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941474.464146:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941474.464150:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941474.464155:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241155200:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941474.464161:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941474.464167:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941474.464170:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941474.464173:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941474.464179:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735400. 02000000:00000001:1.0:1773941474.464182:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941474.464197:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941474.464202:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941474.464211:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941474.464220:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241155200, portal 25 00000100:00000001:1.0:1773941474.464222:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941474.464224:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121069568 : -128249588482048 : ffff8b5b90d60000) 00000100:00000040:1.0:1773941474.464232:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60000 x1860112241155200/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941490 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941474.464243:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941474.464246:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941474.464290:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1770. 00000100:00000200:1.0:1773941474.464296:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241155200, offset 0 00000400:00000200:1.0:1773941474.464330:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941474.464339:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941474.464347:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941474.464352:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941474.464356:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941474.464361:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941474.464365:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941474.464368:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[86, 85] 00000400:00000200:1.0:1773941474.464373:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941474.464377:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [192:192:256:4294967295] 192.168.204.153@tcp LPNI seq info [87:87:8:4294967295] 00000400:00000200:1.0:1773941474.464381:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941474.464393:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941474.464399:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941474.464405:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024be00. 00000800:00000200:1.0:1773941474.464412:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941474.464420:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941474.464425:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024be00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941474.464441:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941474.464446:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941474.464455:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294914827 00000100:00000001:1.0:1773941474.464462:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294913229 00000100:00000001:1.0:1773941474.464467:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294914827 00000100:00000001:1.0:1773941474.464471:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294913229 00000100:00000001:1.0:1773941474.464476:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941474.464479:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941474.464481:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941474.464484:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941474.464487:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941474.464490:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941474.464493:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941474.464565:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941474.464571:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024be00. 00000400:00000200:3.0:1773941474.464577:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941474.464584:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941474.464589:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1770 00000400:00000010:3.0:1773941474.464592:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1770. 00000100:00000001:3.0:1773941474.464596:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941474.464602:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60000 x1860112241155200/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941490 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941474.464610:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941474.464612:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d60000 x1860112241155200/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941490 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941474.464617:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941474.464619:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941474.465148:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941474.465157:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941474.465159:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941474.465167:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941474.465171:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941474.465174:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941474.465179:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941474.465181:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941474.465184:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941474.465187:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605c80 00000400:00000200:3.0:1773941474.465190:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941474.465193:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xeed [1] + 224 00000800:00000001:3.0:1773941474.465199:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941474.465204:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941474.465205:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941474.465208:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941474.465211:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941474.465215:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60000 x1860112241155200/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941490 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941474.465221:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d60000 x1860112241155200/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941490 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941474.465234:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941474.465239:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941474.465242:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941474.465276:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941474.465282:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941474.465283:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941474.465284:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941474.465287:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941474.465289:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941474.465290:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941474.465305:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60000 x1860112241155200/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941490 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941474.465313:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60000 x1860112241155200/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941490 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941474.465318:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941474.465320:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941474.465322:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941474.465324:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941474.465325:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941474.465327:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941474.465329:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941474.465333:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=53 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941474.465336:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941474.465338:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941474.465342:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941474.465345:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941474.465354:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941474.465358:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941474.465362:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941474.465367:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941474.465370:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941474.465373:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941474.465374:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941474.465377:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941474.465381:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d60000 x1860112241155200/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941490 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941474.465390:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941474.465392:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941474.465394:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941474.465396:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941474.465400:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941474.465401:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941474.465403:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941474.465404:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941474.465406:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941474.465408:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941474.465409:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941474.465412:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941474.465414:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941474.465417:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941474.465420:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 243 00000400:00000200:1.0:1773941474.465425:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941474.465430:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941474.465434:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941474.465437:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941474.465439:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941474.465442:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941474.465445:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941474.465452:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941474.465455:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941474.465458:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941474.465460:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941474.465463:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941474.465467:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941474.465469:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941474.465471:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941474.465473:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941474.465476:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941474.465480:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941474.465483:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941474.465485:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941474.465488:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941474.465491:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941474.465493:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941474.465496:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941474.465499:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63480. 00000020:00000040:1.0:1773941474.465502:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941474.465505:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941474.465507:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941474.465509:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941474.465511:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941474.465513:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941474.465517:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734800. 00000100:00000001:1.0:1773941474.465522:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941474.465527:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63480 x1860112241155328/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941474.465536:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941474.465539:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941474.465544:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000834a8cee] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63480 x1860112241155328/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941474.465578:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941474.465581:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941474.465583:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941474.465587:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941474.465589:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941474.465591:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941474.465593:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941474.465595:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941474.465598:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941474.465605:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60000 x1860112241155200/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941490 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941474.465613:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941474.465614:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941474.465618:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941474.465619:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941474.465621:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941474.465625:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:2.0:1773941474.465629:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63480 x1860112241155328/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941474.465630:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941474.465640:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1773941474.465641:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60000 x1860112241155200/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941490 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941474.465643:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941474.465647:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941474.465649:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941474.465653:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941474.465659:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241155328:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941474.465664:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1773941474.465665:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241155200:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941474.465672:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941474.465673:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941474.465676:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941474.465677:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941474.465678:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941474.465683:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000010:2.0:1773941474.465684:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce4400. 00000100:00000001:1.0:1773941474.465686:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941474.465687:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941474.465693:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000040:1.0:1773941474.465694:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60000 x1860112241155200/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941490 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941474.465696:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941474.465724:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000200:2.0:1773941474.465728:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241155328, portal 25 00000100:00000001:2.0:1773941474.465730:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941474.465731:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083008 : -128249588468608 : ffff8b5b90d63480) 00000100:00000040:2.0:1773941474.465741:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63480 x1860112241155328/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941474.465747:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941474.465750:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941474.465753:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34330. 00000100:00000200:2.0:1773941474.465756:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241155328, offset 0 00000400:00000200:2.0:1773941474.465763:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000100:00000001:1.0:1773941474.465764:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941474.465769:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941474.465770:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000400:00000200:2.0:1773941474.465774:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000100:00000001:1.0:1773941474.465776:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000400:00000200:2.0:1773941474.465778:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1773941474.465782:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941474.465783:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941474.465787:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941474.465789:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000200:2.0:1773941474.465790:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941474.465792:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[86, 87] 00000400:00000200:2.0:1773941474.465794:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941474.465797:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [193:193:256:4294967295] 192.168.204.143@tcp LPNI seq info [88:88:8:4294967295] 00010000:00000001:1.0:1773941474.465801:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941474.465802:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:1.0:1773941474.465807:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941474.465812:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000020:00000040:1.0:1773941474.465812:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 00000800:00000200:2.0:1773941474.465819:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941474.465823:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0600. 00000800:00000200:2.0:1773941474.465826:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000020:00000001:1.0:1773941474.465831:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941474.465838:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000800:00000200:2.0:1773941474.465839:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941474.465843:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0600 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:1.0:1773941474.465846:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d40 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941474.465856:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941474.465857:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941474.465862:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941474.465864:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294914829 00000100:00000001:2.0:1773941474.465867:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294914827 00010000:00010000:1.0:1773941474.465868:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d40 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941474.465871:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294914829 00000100:00000001:2.0:1773941474.465886:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294914827 00000100:00000001:2.0:1773941474.465890:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941474.465894:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:2.0:1773941474.465895:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941474.465899:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941474.465901:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d40 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941474.465902:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941474.465910:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941474.465912:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941474.465913:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941474.465918:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1773941474.465918:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941474.465919:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941474.465923:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941474.465925:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941474.465925:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d40 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941474.465935:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941474.465938:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941474.465953:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941474.465957:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d40 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941474.465964:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941474.465967:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941474.465970:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d40 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941474.465990:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941474.465990:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941474.465995:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000800:00000010:3.0:1773941474.465996:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0600. 00000020:00000001:1.0:1773941474.465998:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:3.0:1773941474.466002:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:1.0:1773941474.466002:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000400:00000200:3.0:1773941474.466009:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000020:00000001:1.0:1773941474.466009:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000400:00000200:3.0:1773941474.466015:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34330 00010000:00000001:1.0:1773941474.466016:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000400:00000010:3.0:1773941474.466018:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34330. 00000100:00000001:3.0:1773941474.466023:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941474.466023:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941474.466030:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d40 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941474.466031:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63480 x1860112241155328/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941484 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941474.466050:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00000001:1.0:1773941474.466053:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941474.466056:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d40 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941474.466074:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941474.466076:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941474.466078:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941474.466081:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941474.466083:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941474.466086:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941474.466087:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941474.466089:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941474.466092:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941474.466094:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1d40 from hash 00010000:00000001:1.0:1773941474.466097:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941474.466099:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941474.466101:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941474.466104:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000040:3.0:1773941474.466107:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63480 x1860112241155328/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941484 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941474.466108:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941474.466111:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941474.466114:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941474.466118:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941474.466121:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941474.466133:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941474.466138:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d40 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941474.466154:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941474.466156:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941474.466157:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1773941474.466161:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941474.466165:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941474.466167:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941474.466197:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941474.466199:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941474.466201:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941474.466205:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d40 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941474.466215:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890eca00 count: 0 00010000:00000001:1.0:1773941474.466230:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941474.466233:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941474.466237:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941474.466245:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60000 x1860112241155200/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941490 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941474.466287:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941474.466290:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941474.466293:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735400. 02000000:00000001:1.0:1773941474.466297:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941474.466299:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941474.466301:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941474.466304:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941474.466307:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941474.466310:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941474.466311:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941474.466313:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60000. 00000100:00000001:1.0:1773941474.466316:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941474.466318:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941474.466320:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941474.466323:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941474.466325:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941474.466334:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941474.466339:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941474.466343:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941474.466348:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941474.466352:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941474.466357:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941474.466362:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941474.466367:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941474.466372:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941474.466377:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941474.466379:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941474.466384:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941474.466388:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941474.466391:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941474.466396:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941474.466399:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941474.466403:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941474.466407:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941474.466410:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941474.466412:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941474.466415:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941474.466418:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941474.466421:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941474.466423:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941474.466424:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941474.466426:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941474.466428:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941474.467388:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941474.467395:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941474.467407:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941474.467409:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941474.467414:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941474.467435:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941474.467482:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941474.467486:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941474.467489:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941474.467493:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605d00 00000400:00000200:3.0:1773941474.467497:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941474.467500:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xef5 [1] + 0 00000800:00000001:3.0:1773941474.467504:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941474.467509:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941474.467511:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941474.467514:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941474.467517:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941474.467520:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63480 x1860112241155328/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941484 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941474.467528:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63480 x1860112241155328/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941484 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941474.467543:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941474.467563:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941474.467565:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941474.467598:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941474.467602:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941474.467606:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941474.467608:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941474.467611:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941474.467616:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941474.467618:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941474.467622:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941474.467644:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63480 x1860112241155328/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941484 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941474.467654:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63480 x1860112241155328/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941484 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941474.467660:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941474.467661:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941474.467663:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941474.467670:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941474.467671:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941474.467672:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941474.467674:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941474.467677:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=53 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941474.467679:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941474.467681:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941474.467682:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941474.467684:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941474.467685:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941474.467687:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941474.467690:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63480 x1860112241155328/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941484 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941474.467695:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941474.467697:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941474.467698:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941474.467700:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941474.467703:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941474.467704:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941474.467706:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941474.467707:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941474.467709:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941474.467710:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941474.467712:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941474.467715:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941474 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941474.467717:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941474.467737:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941474.467739:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941474.467741:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941474.467742:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941474.467744:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941474.467746:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941474.467749:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941474.467751:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941474.467758:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941474.467761:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941474.467763:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941474.467972:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941474.467975:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941474.467979:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63480 x1860112241155328/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941484 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941474.467986:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241155328:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941474.467990:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941474.467992:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941474.467995:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63480 x1860112241155328/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941484 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941474.467995:0:12198:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941474.468003:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941474.468005:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941474.468007:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce4400. 00000100:00080000:1.0:1773941474.468009:0:12198:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 02000000:00000001:2.0:1773941474.468010:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941474.468013:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941474.468015:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00100000:1.0:1773941474.468016:0:12198:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000001:2.0:1773941474.468017:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941474.468020:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734800. 00000100:00000001:1.0:1773941474.468022:0:12198:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941474.468023:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:01000000:1.0:1773941474.468025:0:12198:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941474.468026:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941474.468028:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63480. 00000100:00000001:2.0:1773941474.468032:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:1.0:1773941474.468032:0:12198:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941474.468035:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941474.468038:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941474.468038:0:12198:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941474.468042:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941474.468043:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941474.468047:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941474.468048:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941474.468050:0:12198:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941474.468054:0:12198:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941474.468055:0:12198:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941474.468058:0:12198:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941474.468060:0:12198:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941474.468062:0:12198:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941474.468066:0:12198:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941474.468069:0:12198:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941474.468071:0:12198:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941474.468074:0:12198:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941474.468076:0:12198:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941474.468078:0:12198:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941474.468080:0:12198:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941474.468082:0:12198:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941474.468099:0:12198:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941474.468101:0:12198:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941474.468103:0:12198:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941474.468107:0:12198:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941474.468112:0:12198:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941474.468115:0:12198:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941474.468117:0:12198:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941474.468121:0:12198:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941474.468123:0:12198:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941474.471731:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890eca00. 00010000:00000010:1.1:1773941474.471737:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:0.0:1773941475.167888:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941475.167896:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941475.167901:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941476.191802:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941476.191808:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941476.191812:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941476.191815:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941476.191817:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941476.191818:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941476.191834:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941476.191836:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941476.191838:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941476.191840:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941476.191842:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941476.191849:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277b80. 00000020:00000001:0.0:1773941476.191851:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000040:3.0:1773941476.191853:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941476.191855:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941476.191859:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00010000:00010000:0.0:1773941476.191859:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 02000000:00000001:3.0:1773941476.191863:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941476.191869:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941476.191875:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e3a00. 00000100:00000001:3.0:1773941476.191880:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941476.191884:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941476.191886:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941476.191892:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962400. 00010000:00000001:3.0:1773941476.191896:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941476.191899:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81484400. 00000020:00000001:3.0:1773941476.191902:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941476.191904:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81484400 with handle 0xa33a5be14eae1d47 to hash 00000020:00000001:3.0:1773941476.191906:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941476.191907:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860122624 : -128249849428992 : ffff8b5b81484400) 00010000:00000001:3.0:1773941476.191910:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860122624 : -128249849428992 : ffff8b5b81484400) 00010000:00000001:3.0:1773941476.191913:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941476.191915:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941476.191918:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1d47 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941476.191926:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1d47 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941476.191934:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1d47 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941476.191938:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941476.191943:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941476.191953:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e300. 00000100:00000001:3.0:1773941476.191955:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387400448 : -128249322151168 : ffff8b5ba0b5e300) 00000100:00000001:3.0:1773941476.191957:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941476.191958:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109632384 : -128249599919232 : ffff8b5b90277b80) 00000100:00000001:3.0:1773941476.191962:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941476.191966:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941476.191967:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941476.191968:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941476.191977:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90277b80 x1860112241155456/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941476.191984:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941476.191985:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941476.191988:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941476.191990:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941476.191992:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941476.191994:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241155456:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941476.191997:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941476.192000:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941476.192002:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941476.192004:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941476.192008:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90263000. 02000000:00000001:3.0:1773941476.192009:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941476.192014:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941476.192017:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abec8. 00000400:00000010:3.0:1773941476.192022:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a440. 00000100:00000200:3.0:1773941476.192027:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241155456, portal 25 00000100:00000001:3.0:1773941476.192028:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941476.192030:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109632384 : -128249599919232 : ffff8b5b90277b80) 00000100:00000040:3.0:1773941476.192033:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90277b80 x1860112241155456/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941492 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941476.192038:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941476.192041:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941476.192046:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6aa18. 00000100:00000200:3.0:1773941476.192048:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241155456, offset 0 00000400:00000200:3.0:1773941476.192052:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941476.192059:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941476.192065:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941476.192069:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941476.192072:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941476.192076:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941476.192078:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941476.192080:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[88, 87] 00000400:00000200:3.0:1773941476.192082:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941476.192085:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [194:194:256:4294967295] 192.168.204.153@tcp LPNI seq info [89:89:8:4294967295] 00000400:00000200:3.0:1773941476.192088:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941476.192095:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941476.192099:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941476.192103:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69c00. 00000800:00000200:3.0:1773941476.192108:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941476.192114:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941476.192117:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69c00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941476.192131:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941476.192135:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941476.192138:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294916555 00000100:00000001:3.0:1773941476.192146:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294914829 00000100:00000001:3.0:1773941476.192156:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294916555 00000100:00000001:3.0:1773941476.192160:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294914829 00000100:00000001:3.0:1773941476.192163:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941476.192166:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941476.192168:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941476.192170:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941476.192173:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 11 seconds 00000100:00000001:3.0:1773941476.192185:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941476.192188:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941476.192256:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941476.192260:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69c00. 00000400:00000200:2.0:1773941476.192265:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941476.192273:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941476.192278:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6aa18 00000400:00000010:2.0:1773941476.192280:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6aa18. 00000100:00000001:2.0:1773941476.192284:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941476.192288:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277b80 x1860112241155456/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941492 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941476.192296:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941476.192299:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90277b80 x1860112241155456/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941492 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941476.192304:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941476.192305:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941476.192938:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941476.192948:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941476.192951:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941476.192953:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941476.192959:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941476.192964:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941476.192970:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941476.192974:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941476.193033:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941476.193037:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605d80 00000400:00000200:3.0:1773941476.193042:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941476.193047:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xf05 [1] + 224 00000800:00000001:3.0:1773941476.193054:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941476.193062:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941476.193065:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941476.193068:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941476.193073:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941476.193078:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277b80 x1860112241155456/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941492 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941476.193089:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90277b80 x1860112241155456/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941492 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941476.193108:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941476.193115:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941476.193119:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941476.193140:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941476.193146:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941476.193148:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941476.193150:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abec8. 00000400:00000200:1.0:1773941476.193154:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a440 00000400:00000010:1.0:1773941476.193155:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a440. 00000100:00000001:1.0:1773941476.193158:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941476.193162:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277b80 x1860112241155456/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941492 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941476.193171:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277b80 x1860112241155456/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941492 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941476.193176:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941476.193184:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941476.193186:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941476.193188:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941476.193190:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941476.193192:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941476.193193:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941476.193198:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=55 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941476.193200:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941476.193203:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941476.193206:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941476.193208:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941476.193211:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941476.193213:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941476.193231:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941476.193235:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941476.193238:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941476.193239:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941476.193241:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941476.193242:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941476.193246:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90277b80 x1860112241155456/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941492 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941476.193251:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941476.193252:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941476.193253:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941476.193255:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941476.193257:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941476.193258:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941476.193259:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941476.193260:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941476.193261:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941476.193262:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941476.193263:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941476.193265:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941476.193266:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941476.193273:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941476.193276:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 244 00000400:00000200:1.0:1773941476.193279:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941476.193282:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941476.193285:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941476.193288:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941476.193290:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941476.193292:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941476.193294:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941476.193296:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941476.193297:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941476.193299:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941476.193301:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941476.193302:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941476.193304:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941476.193305:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941476.193306:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941476.193307:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941476.193309:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941476.193312:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941476.193313:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941476.193316:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941476.193318:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941476.193320:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941476.193322:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941476.193323:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941476.193327:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60000. 00000020:00000040:1.0:1773941476.193328:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941476.193331:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941476.193332:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941476.193333:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941476.193335:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941476.193339:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941476.193343:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0735400. 00000100:00000001:1.0:1773941476.193346:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941476.193350:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60000 x1860112241155584/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941476.193356:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941476.193358:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941476.193361:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000b65cec4] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60000 x1860112241155584/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941476.193379:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941476.193383:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941476.193384:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941476.193387:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941476.193389:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941476.193391:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941476.193393:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 1 seconds 00000100:00000001:1.0:1773941476.193395:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941476.193397:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941476.193402:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90277b80 x1860112241155456/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941492 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941476.193410:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941476.193413:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941476.193416:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941476.193418:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941476.193420:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1773941476.193425:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90277b80 x1860112241155456/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941492 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941476.193427:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941476.193431:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941476.193441:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60000 x1860112241155584/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941476.193444:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241155456:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941476.193453:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941476.193459:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941476.193461:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941476.193463:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941476.193465:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941476.193468:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941476.193470:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941476.193473:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941476.193475:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941476.193481:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90277b80 x1860112241155456/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941492 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941476.193483:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241155584:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941476.193493:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941476.193497:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941476.193500:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941476.193502:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941476.193504:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:1.0:1773941476.193510:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e300. 02000000:00000010:2.0:1773941476.193511:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce5400. 00000100:00000001:1.0:1773941476.193514:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:2.0:1773941476.193516:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941476.193518:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941476.193524:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000200:2.0:1773941476.193525:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941476.193528:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000010:2.0:1773941476.193530:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000020:00000001:1.0:1773941476.193531:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941476.193534:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484400 refcount=4 00000020:00000001:1.0:1773941476.193537:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860122624 : -128249849428992 : ffff8b5b81484400) 00000400:00000010:2.0:1773941476.193538:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00000001:1.0:1773941476.193539:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860122624 : -128249849428992 : ffff8b5b81484400) 00000100:00000200:2.0:1773941476.193546:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241155584, portal 25 00010000:00010000:1.0:1773941476.193546:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1d47 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941476.193552:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941476.193556:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941476.193559:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1d47 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941476.193562:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121069568 : -128249588482048 : ffff8b5b90d60000) 00010000:00000001:1.0:1773941476.193569:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941476.193573:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1d47 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941476.193574:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60000 x1860112241155584/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941486 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941476.193585:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941476.193589:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941476.193591:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1d47 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941476.193596:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941476.193601:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000040:2.0:1773941476.193604:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941476.193604:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941476.193609:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000010:2.0:1773941476.193610:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34088. 00000100:00000200:2.0:1773941476.193615:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241155584, offset 0 00010000:00010000:1.0:1773941476.193619:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1d47 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941476.193626:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941476.193631:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81484400 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941476.193635:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941476.193636:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00010000:1.0:1773941476.193638:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1d47 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941476.193645:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:1.0:1773941476.193645:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941476.193647:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941476.193650:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941476.193652:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:1.0:1773941476.193653:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484400 refcount=5 00000020:00000001:1.0:1773941476.193658:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860122624 : -128249849428992 : ffff8b5b81484400) 00000400:00000200:2.0:1773941476.193661:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941476.193661:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860122624 : -128249849428992 : ffff8b5b81484400) 00010000:00000001:1.0:1773941476.193666:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:2.0:1773941476.193669:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:1.0:1773941476.193670:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1d47 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941476.193677:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000001:1.0:1773941476.193679:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941476.193683:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1d47 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941476.193684:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[88, 89] 10000000:01000000:1.0:1773941476.193694:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941476.193698:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 10000000:00000040:1.0:1773941476.193699:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941476.193702:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941476.193724:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941476.193725:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941476.193728:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941476.193729:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941476.193730:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941476.193732:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941476.193735:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81484400 with handle 0xa33a5be14eae1d47 from hash 00000400:00000200:2.0:1773941476.193737:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [195:195:256:4294967295] 192.168.204.143@tcp LPNI seq info [90:90:8:4294967295] 00010000:00000001:1.0:1773941476.193738:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941476.193745:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941476.193746:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941476.193747:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941476.193749:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941476.193752:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941476.193754:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941476.193759:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941476.193760:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941476.193763:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941476.193764:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 10000000:00000001:1.0:1773941476.193765:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941476.193771:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1d47 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941476.193777:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941476.193780:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941476.193783:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941476.193784:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000800:00000010:2.0:1773941476.193785:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0500. 00010000:00000001:1.0:1773941476.193786:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941476.193788:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941476.193791:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941476.193792:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941476.193795:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81484400/0xa33a5be14eae1d47 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941476.193796:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000040:1.0:1773941476.193807:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962400 count: 0 00000800:00000200:2.0:1773941476.193812:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941476.193817:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0500 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941476.193817:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941476.193820:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941476.193823:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941476.193828:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90277b80 x1860112241155456/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941492 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941476.193832:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941476.193834:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941476.193836:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941476.193837:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90263000. 02000000:00000001:1.0:1773941476.193842:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941476.193845:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:2.0:1773941476.193846:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000040:1.0:1773941476.193847:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941476.193852:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294916557 00000020:00000001:1.0:1773941476.193852:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941476.193856:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e3a00. 00000100:00000001:2.0:1773941476.193860:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294916555 02000000:00000001:1.0:1773941476.193865:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1773941476.193867:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294916557 02000000:00000001:1.0:1773941476.193868:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941476.193872:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277b80. 00000100:00000001:2.0:1773941476.193874:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294916555 00000100:00000001:1.0:1773941476.193876:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941476.193877:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941476.193879:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941476.193880:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941476.193882:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941476.193887:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000100:00000001:2.0:1773941476.193889:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1773941476.193892:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000800:00000200:3.0:1773941476.193894:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941476.193894:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941476.193898:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000800:00000010:3.0:1773941476.193900:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0500. 00000100:00000001:2.0:1773941476.193901:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:1.0:1773941476.193906:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000100:00000001:2.0:1773941476.193907:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941476.193908:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:1.0:1773941476.193914:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000400:00000200:3.0:1773941476.193917:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941476.193918:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:1.0:1773941476.193922:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 00000400:00000200:3.0:1773941476.193923:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34088 00000100:00000001:2.0:1773941476.193923:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000010:3.0:1773941476.193930:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34088. 00000100:00000001:2.0:1773941476.193930:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 80000000:00000010:1.0:1773941476.193932:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000001:3.0:1773941476.193935:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1773941476.193935:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941476.193939:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:1.0:1773941476.193940:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 00000100:00000001:2.0:1773941476.193942:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1773941476.193948:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60000 x1860112241155584/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941486 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 20000000:00000010:1.0:1773941476.193950:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 00000100:00000001:3.0:1773941476.193959:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00000040:1.0:1773941476.193959:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941476.193964:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941476.193968:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941476.193971:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941476.193973:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941476.193976:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941476.193978:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941476.193982:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941476.193984:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941476.193986:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941476.193988:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941476.193990:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941476.193992:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941476.193994:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941476.193995:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941476.193996:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941476.193997:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941476.194004:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000040:3.0:1773941476.194016:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60000 x1860112241155584/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941486 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941476.194022:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941476.194024:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941476.195240:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941476.195246:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941476.195248:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941476.195250:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941476.195253:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941476.195257:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941476.195261:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941476.195263:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941476.195266:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941476.195268:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605e00 00000400:00000200:3.0:1773941476.195271:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941476.195274:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xf15 [1] + 0 00000800:00000001:3.0:1773941476.195277:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941476.195281:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941476.195283:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941476.195285:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941476.195288:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941476.195292:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60000 x1860112241155584/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941486 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941476.195300:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60000 x1860112241155584/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941486 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941476.195312:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941476.195346:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941476.195350:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000800:00000001:3.0:1773941476.195351:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000400:00000001:2.0:1773941476.195354:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000001:3.0:1773941476.195356:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:2.0:1773941476.195358:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941476.195362:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941476.195367:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941476.195369:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941476.195373:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941476.195378:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60000 x1860112241155584/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941486 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941476.195389:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60000 x1860112241155584/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941486 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941476.195396:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941476.195398:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941476.195400:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941476.195402:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941476.195404:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941476.195406:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941476.195408:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941476.195411:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=55 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941476.195414:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941476.195417:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941476.195418:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941476.195421:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941476.195424:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941476.195426:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941476.195438:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60000 x1860112241155584/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941486 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941476.195445:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941476.195447:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941476.195449:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941476.195451:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941476.195456:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941476.195458:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941476.195460:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941476.195462:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941476.195464:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941476.195466:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941476.195469:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941476.195472:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941476 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941476.195475:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941476.195481:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941476.195484:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941476.195485:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941476.195487:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941476.195489:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941476.195492:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941476.195494:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941476.195497:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941476.195500:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941476.195504:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941476.195506:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941476.195735:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941476.195739:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941476.195743:0:12199:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941476.195747:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60000 x1860112241155584/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941486 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941476.195753:0:12199:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:1.0:1773941476.195760:0:12199:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:2.0:1773941476.195767:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241155584:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941476.195786:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941476.195789:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941476.195791:0:12199:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941476.195794:0:12199:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000040:2.0:1773941476.195795:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60000 x1860112241155584/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941486 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:1.0:1773941476.195797:0:12199:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941476.195801:0:12199:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941476.195804:0:12199:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941476.195808:0:12199:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941476.195811:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:1.0:1773941476.195811:0:12199:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941476.195813:0:12199:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941476.195815:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 10000000:00000001:1.0:1773941476.195816:0:12199:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941476.195820:0:12199:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:2.0:1773941476.195821:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce5400. 00000100:00080000:1.0:1773941476.195823:0:12199:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941476.195827:0:12199:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:2.0:1773941476.195828:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941476.195829:0:12199:0:(recover.c:147:ptlrpc_resend()) Process entered 00000020:00000001:2.0:1773941476.195831:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941476.195832:0:12199:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941476.195834:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00080000:1.0:1773941476.195836:0:12199:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000020:00000001:2.0:1773941476.195840:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941476.195840:0:12199:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:2.0:1773941476.195844:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0735400. 10000000:01000000:1.0:1773941476.195844:0:12199:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941476.195847:0:12199:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 02000000:00000001:2.0:1773941476.195852:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941476.195855:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941476.195856:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60000. 00000100:00000001:2.0:1773941476.195860:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:1.0:1773941476.195861:0:12199:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:2.0:1773941476.195863:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941476.195864:0:12199:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941476.195868:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941476.195868:0:12199:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941476.195875:0:12199:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941476.195876:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941476.195878:0:12199:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941476.195880:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941476.195881:0:12199:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941476.195886:0:12199:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941476.195887:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1773941476.195888:0:12199:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941476.195891:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941476.195892:0:12199:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941476.199738:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962400. 00010000:00000010:1.1:1773941476.199746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81484400. 00000020:00000001:0.0:1773941477.215901:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941477.215908:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941477.215913:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941477.471806:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941477.471831:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941477.471836:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941477.471839:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941477.471841:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941477.471842:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941477.471846:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941477.471848:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941477.471851:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941477.471854:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941477.471855:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941477.471864:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90274700. 00000020:00000040:3.0:1773941477.471869:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941477.471874:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941477.471877:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941477.471882:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941477.471890:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2600. 00000100:00000001:3.0:1773941477.471896:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941477.471901:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941477.471904:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941477.471912:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962a00. 00010000:00000001:3.0:1773941477.471918:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941477.471921:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485600. 00000020:00000001:3.0:1773941477.471926:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941477.471929:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485600 with handle 0xa33a5be14eae1d4e to hash 00000020:00000001:3.0:1773941477.471931:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941477.471932:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860127232 : -128249849424384 : ffff8b5b81485600) 00010000:00000001:3.0:1773941477.471936:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860127232 : -128249849424384 : ffff8b5b81485600) 00010000:00000001:3.0:1773941477.471940:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941477.471944:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941477.471948:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1d4e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941477.471960:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1d4e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941477.471986:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1d4e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941477.471993:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941477.471995:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941477.472001:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941477.472003:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941477.472006:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941477.472007:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109618944 : -128249599932672 : ffff8b5b90274700) 00000100:00000001:3.0:1773941477.472012:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941477.472016:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941477.472019:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941477.472021:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941477.472031:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90274700 x1860112241155712/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941477.472041:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941477.472043:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941477.472047:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941477.472050:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941477.472053:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941477.472057:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90274700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241155712:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941477.472061:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941477.472066:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941477.472068:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941477.472071:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941477.472077:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90262000. 02000000:00000001:3.0:1773941477.472080:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941477.472087:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941477.472091:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abdc0. 00000400:00000010:3.0:1773941477.472099:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6aee0. 00000100:00000200:3.0:1773941477.472106:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241155712, portal 25 00000100:00000001:3.0:1773941477.472108:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941477.472115:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109618944 : -128249599932672 : ffff8b5b90274700) 00000100:00000040:3.0:1773941477.472122:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90274700 x1860112241155712/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941493 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941477.472130:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941477.472134:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941477.472137:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a880. 00000100:00000200:3.0:1773941477.472140:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241155712, offset 0 00000400:00000200:3.0:1773941477.472148:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941477.472157:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941477.472164:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941477.472170:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941477.472174:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941477.472178:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941477.472181:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941477.472184:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[90, 89] 00000400:00000200:3.0:1773941477.472188:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941477.472212:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [196:196:256:4294967295] 192.168.204.153@tcp LPNI seq info [91:91:8:4294967295] 00000400:00000200:3.0:1773941477.472218:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941477.472228:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941477.472234:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941477.472240:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69700. 00000800:00000200:3.0:1773941477.472248:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941477.472256:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941477.472260:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69700 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941477.472283:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941477.472288:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941477.472291:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294917835 00000100:00000001:3.0:1773941477.472298:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294916557 00000100:00000001:3.0:1773941477.472303:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294917835 00000100:00000001:3.0:1773941477.472308:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294916557 00000100:00000001:3.0:1773941477.472318:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941477.472321:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941477.472324:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941477.472326:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941477.472329:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941477.472333:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941477.472335:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941477.472415:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941477.472421:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69700. 00000400:00000200:2.0:1773941477.472427:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941477.472433:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941477.472437:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a880 00000400:00000010:2.0:1773941477.472439:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a880. 00000100:00000001:2.0:1773941477.472444:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941477.472452:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90274700 x1860112241155712/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941493 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941477.472461:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941477.472464:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90274700 x1860112241155712/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941493 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941477.472479:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941477.472481:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941477.473062:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941477.473081:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941477.473122:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941477.473125:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941477.473133:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941477.473140:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941477.473149:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941477.473155:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941477.473161:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941477.473167:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605e80 00000400:00000200:3.0:1773941477.473172:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941477.473179:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xf25 [1] + 224 00000800:00000001:3.0:1773941477.473189:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941477.473215:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941477.473219:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941477.473224:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941477.473231:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941477.473240:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90274700 x1860112241155712/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941493 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941477.473253:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90274700 x1860112241155712/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941493 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941477.473273:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941477.473283:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941477.473287:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941477.473310:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941477.473321:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941477.473324:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941477.473328:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abdc0. 00000400:00000200:1.0:1773941477.473333:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6aee0 00000400:00000010:1.0:1773941477.473349:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6aee0. 00000100:00000001:1.0:1773941477.473354:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941477.473360:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90274700 x1860112241155712/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941493 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941477.473372:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90274700 x1860112241155712/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941493 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941477.473381:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941477.473383:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941477.473386:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941477.473391:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941477.473394:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941477.473396:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941477.473400:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941477.473409:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=56 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941477.473414:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941477.473417:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941477.473424:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941477.473428:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941477.473432:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941477.473435:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941477.473441:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941477.473446:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941477.473450:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941477.473453:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941477.473455:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941477.473457:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941477.473462:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90274700 x1860112241155712/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941493 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941477.473472:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941477.473473:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941477.473476:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941477.473487:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941477.473491:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941477.473493:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941477.473496:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941477.473497:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941477.473498:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941477.473500:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941477.473501:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941477.473504:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941477.473507:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941477.473520:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941477.473523:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 246 00000400:00000200:1.0:1773941477.473527:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941477.473531:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941477.473536:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941477.473539:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941477.473542:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941477.473545:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941477.473549:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941477.473551:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941477.473554:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941477.473557:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941477.473559:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941477.473562:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941477.473566:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941477.473568:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941477.473570:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941477.473572:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941477.473576:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941477.473580:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941477.473584:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941477.473586:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941477.473590:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941477.473599:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941477.473602:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941477.473604:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941477.473609:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941477.473612:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941477.473616:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941477.473617:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941477.473620:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941477.473622:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941477.473625:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941477.473630:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737000. 00000100:00000001:1.0:1773941477.473635:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941477.473640:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63800 x1860112241155840/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941477.473650:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941477.473654:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941477.473660:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc928f9c] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63800 x1860112241155840/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941477.473684:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941477.473687:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941477.473689:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941477.473693:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941477.473696:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941477.473698:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941477.473701:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941477.473732:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941477.473735:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941477.473738:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941477.473745:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1773941477.473745:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90274700 x1860112241155712/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941493 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941477.473751:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941477.473765:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241155840/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941477.473768:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941477.473778:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941477.473787:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941477.473794:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941477.473796:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941477.473798:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941477.473802:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941477.473807:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941477.473813:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941477.473819:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90274700 x1860112241155712/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941493 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941477.473820:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241155840:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941477.473839:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941477.473846:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:1.0:1773941477.473851:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90274700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241155712:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941477.473852:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941477.473858:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941477.473865:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce5c00. 00000100:00000001:1.0:1773941477.473866:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941477.473874:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941477.473882:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941477.473887:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941477.473889:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941477.473896:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000010:2.0:1773941477.473897:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941477.473907:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000040:1.0:1773941477.473908:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90274700 x1860112241155712/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941493 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941477.473917:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241155840, portal 25 00000100:00000001:1.0:1773941477.473919:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941477.473923:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000010:1.0:1773941477.473925:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:2.0:1773941477.473927:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000001:1.0:1773941477.473930:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941477.473939:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:2.0:1773941477.473942:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241155840/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941477.473946:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941477.473952:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941477.473955:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941477.473959:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:1.0:1773941477.473961:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485600 refcount=4 00000100:00000040:2.0:1773941477.473970:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000020:00000001:1.0:1773941477.473970:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860127232 : -128249849424384 : ffff8b5b81485600) 00000400:00000010:2.0:1773941477.473975:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f343b8. 00010000:00000001:1.0:1773941477.473978:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860127232 : -128249849424384 : ffff8b5b81485600) 00000100:00000200:2.0:1773941477.473982:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241155840, offset 0 00010000:00010000:1.0:1773941477.473987:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1d4e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941477.473995:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941477.474005:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941477.474015:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1d4e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941477.474016:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941477.474026:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941477.474030:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941477.474036:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941477.474039:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:2.0:1773941477.474043:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:1.0:1773941477.474045:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1d4e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941477.474054:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941477.474066:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[90, 91] 00000400:00000200:2.0:1773941477.474071:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941477.474082:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [197:197:256:4294967295] 192.168.204.143@tcp LPNI seq info [92:92:8:4294967295] 00010000:00000001:1.0:1773941477.474097:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941477.474102:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941477.474103:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941477.474107:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1d4e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941477.474130:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941477.474132:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941477.474139:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941477.474149:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000800:00000200:2.0:1773941477.474153:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941477.474154:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1d4e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:2.0:1773941477.474167:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0300. 00010000:00010000:1.0:1773941477.474170:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485600 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941477.474176:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941477.474179:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1d4e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941477.474190:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941477.474190:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941477.474209:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941477.474212:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941477.474216:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485600 refcount=5 00000800:00000200:2.0:1773941477.474217:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000020:00000001:1.0:1773941477.474227:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860127232 : -128249849424384 : ffff8b5b81485600) 00000800:00000200:2.0:1773941477.474228:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0300 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941477.474236:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860127232 : -128249849424384 : ffff8b5b81485600) 00010000:00000001:1.0:1773941477.474244:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941477.474248:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1d4e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941477.474250:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941477.474263:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941477.474270:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294917837 10000000:00000001:1.0:1773941477.474274:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:2.0:1773941477.474280:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294917835 10000000:00010000:1.0:1773941477.474281:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1d4e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941477.474291:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294917837 10000000:01000000:1.0:1773941477.474300:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:2.0:1773941477.474301:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294917835 00000100:00000001:2.0:1773941477.474305:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941477.474307:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941477.474318:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:1.0:1773941477.474318:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941477.474326:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941477.474327:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:1.0:1773941477.474339:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:2.0:1773941477.474341:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941477.474344:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:2.0:1773941477.474346:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000200:3.0:1773941477.474351:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941477.474351:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941477.474356:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000800:00000010:3.0:1773941477.474359:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0300. 00000100:00000001:2.0:1773941477.474359:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1773941477.474361:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:2.0:1773941477.474364:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941477.474366:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941477.474370:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941477.474374:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485600 with handle 0xa33a5be14eae1d4e from hash 00000400:00000200:3.0:1773941477.474376:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:1.0:1773941477.474379:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941477.474384:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941477.474388:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941477.474389:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941477.474390:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00010000:00000001:1.0:1773941477.474394:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941477.474400:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941477.474403:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:3.0:1773941477.474405:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f343b8 00010000:00000001:1.0:1773941477.474406:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941477.474408:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941477.474411:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941477.474412:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f343b8. 10000000:00000001:1.0:1773941477.474413:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941477.474418:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1d4e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941477.474424:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941477.474424:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941477.474425:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941477.474427:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941477.474428:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941477.474429:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941477.474430:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941477.474432:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485600/0xa33a5be14eae1d4e lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941477.474438:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962a00 count: 0 00010000:00000001:1.0:1773941477.474450:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941477.474452:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941477.474454:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941477.474465:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90274700 x1860112241155712/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941493 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941477.474469:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1773941477.474471:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941477.474475:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000200:3.0:1773941477.474480:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241155840/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1773941477.474480:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90262000. 02000000:00000001:1.0:1773941477.474486:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941477.474489:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941477.474492:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941477.474496:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941477.474500:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000010:1.0:1773941477.474501:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2600. 02000000:00000001:1.0:1773941477.474507:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941477.474509:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:3.0:1773941477.474510:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241155840/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941487 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941477.474511:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90274700. 00000100:00000001:1.0:1773941477.474516:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941477.474517:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941477.474522:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941477.474526:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:3.0:1773941477.474529:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941477.474530:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:3.0:1773941477.474534:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000010:1.0:1773941477.474535:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941477.474541:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941477.474546:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941477.474550:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941477.474555:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941477.474559:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941477.474564:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941477.474568:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941477.474573:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941477.474576:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941477.474578:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941477.474583:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941477.474586:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941477.474588:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941477.474592:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941477.474594:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941477.474601:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941477.474603:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941477.474606:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941477.474608:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941477.474610:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941477.474613:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941477.474615:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941477.474617:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941477.474618:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941477.474620:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941477.474622:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941477.475817:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941477.475822:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941477.475824:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941477.475826:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941477.475830:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941477.475834:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941477.475838:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941477.475840:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941477.475843:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941477.475845:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321605f00 00000400:00000200:3.0:1773941477.475849:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941477.475852:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xf35 [1] + 0 00000800:00000001:3.0:1773941477.475856:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941477.475907:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941477.475909:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941477.475912:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941477.475915:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941477.475918:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241155840/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941487 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941477.475927:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63800 x1860112241155840/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941487 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941477.475945:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941477.475949:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941477.475952:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941477.475978:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941477.475980:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941477.475985:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941477.475986:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941477.475988:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941477.475993:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941477.475995:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941477.475997:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941477.476002:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241155840/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941487 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941477.476010:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241155840/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941487 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941477.476015:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941477.476016:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941477.476018:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941477.476020:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941477.476022:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941477.476023:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941477.476025:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941477.476031:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=56 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941477.476033:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941477.476035:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941477.476036:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941477.476038:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941477.476040:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941477.476042:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941477.476045:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241155840/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941487 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941477.476050:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941477.476051:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941477.476053:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941477.476055:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941477.476059:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941477.476061:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941477.476068:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941477.476070:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941477.476072:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941477.476074:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941477.476076:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941477.476079:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941477 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941477.476081:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941477.476086:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941477.476087:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941477.476090:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941477.476092:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941477.476093:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941477.476095:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941477.476097:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941477.476099:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941477.476101:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941477.476104:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941477.476106:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941477.476376:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941477.476380:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941477.476386:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241155840/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941487 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941477.476395:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241155840:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941477.476400:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941477.476403:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941477.476408:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241155840/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941487 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941477.476414:0:12200:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941477.476416:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941477.476421:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941477.476424:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce5c00. 00000100:00080000:1.0:1773941477.476427:0:12200:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 02000000:00000001:2.0:1773941477.476430:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:1.0:1773941477.476433:0:12200:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941477.476436:0:12200:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941477.476438:0:12200:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000001:2.0:1773941477.476441:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:1.0:1773941477.476443:0:12200:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000040:2.0:1773941477.476445:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:1.0:1773941477.476449:0:12200:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:2.0:1773941477.476450:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941477.476455:0:12200:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941477.476456:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737000. 00000001:00000001:1.0:1773941477.476461:0:12200:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941477.476464:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:1.0:1773941477.476464:0:12200:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941477.476467:0:12200:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941477.476468:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:00000001:1.0:1773941477.476472:0:12200:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941477.476473:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:1.0:1773941477.476478:0:12200:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941477.476480:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941477.476482:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941477.476484:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941477.476485:0:12200:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941477.476488:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941477.476490:0:12200:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941477.476491:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941477.476495:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941477.476496:0:12200:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941477.476497:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941477.476502:0:12200:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941477.476507:0:12200:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941477.476509:0:12200:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941477.476510:0:12200:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941477.476512:0:12200:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941477.476540:0:12200:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941477.476542:0:12200:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941477.476546:0:12200:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941477.476551:0:12200:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941477.476567:0:12200:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941477.476570:0:12200:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941477.476572:0:12200:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941477.476575:0:12200:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941477.476577:0:12200:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941477.479741:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962a00. 00010000:00000010:1.1:1773941477.479752:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485600. 00000020:00000001:0.0:1773941478.239860:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941478.239867:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941478.239872:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941478.303834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1773941478.303843:0:36:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1773941478.303852:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b850aad80. 00000020:00000040:0.0:1773941478.303857:0:36:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941478.303861:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941478.303863:0:36:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941478.303868:0:36:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941478.303876:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b91e1e300. 00000100:00000001:0.0:1773941478.303883:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941478.303894:0:36:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS req@ffff8b5b850aad80 x1860112241155968/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941478.303904:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941478.303907:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941478.303914:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e97c40a7] to pc [ptlrpcd_00_01+1] req@ffff8b5b850aad80 x1860112241155968/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941478.303936:0:36:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941478.303939:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (253) 00000100:00000001:0.0:1773941478.303949:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1773941478.303953:0:2352:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1773941478.303955:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941478.303965:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941478.303985:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941478.303989:0:2352:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941478.303992:0:2352:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941478.303995:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941478.303996:0:2352:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941478.303998:0:2352:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941478.304002:0:2352:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b850aad80 x1860112241155968/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941478.304009:0:2352:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941478.304011:0:2352:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941478.304014:0:2352:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941478.304016:0:2352:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941478.304018:0:2352:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941478.304020:0:2352:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1080e6b5-6a6c-4762-b0ec-c785990fec17:2352:1860112241155968:192.168.204.143@tcp:400:kworker.0 00000100:00000001:0.0:1773941478.304024:0:2352:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941478.304028:0:2352:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941478.304031:0:2352:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941478.304033:0:2352:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941478.304036:0:2352:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b845c4a00. 02000000:00000001:0.0:1773941478.304037:0:2352:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941478.304044:0:2352:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941478.304046:0:2352:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941478.304052:0:2352:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941478.304084:0:2352:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241155968, portal 25 00000100:00000001:0.0:1773941478.304086:0:2352:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941478.304087:0:2352:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615823923195264 : -128249786356352 : ffff8b5b850aad80) 00000100:00000040:0.0:1773941478.304091:0:2352:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b850aad80 x1860112241155968/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941494 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941478.304097:0:2352:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941478.304098:0:2352:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941478.304100:0:2352:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6cee0. 00000100:00000200:0.0:1773941478.304103:0:2352:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241155968, offset 0 00000400:00000200:0.0:1773941478.304108:0:2352:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941478.304114:0:2352:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941478.304120:0:2352:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941478.304128:0:2352:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941478.304131:0:2352:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941478.304134:0:2352:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941478.304136:0:2352:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941478.304139:0:2352:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[92, 91] 00000400:00000200:0.0:1773941478.304141:0:2352:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:0.0:1773941478.304145:0:2352:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [198:198:256:4294967295] 192.168.204.153@tcp LPNI seq info [93:93:8:4294967295] 00000400:00000200:0.0:1773941478.304148:0:2352:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:0.0:1773941478.304157:0:2352:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:0.0:1773941478.304161:0:2352:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:0.0:1773941478.304164:0:2352:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00000800:00000200:0.0:1773941478.304169:0:2352:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:0.0:1773941478.304202:0:2352:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:0.0:1773941478.304207:0:2352:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1773941478.304223:0:2352:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941478.304227:0:2352:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941478.304231:0:2352:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294918667 00000100:00000001:0.0:1773941478.304237:0:2352:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294917837 00000100:00000001:0.0:1773941478.304241:0:2352:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294918667 00000100:00000001:0.0:1773941478.304244:0:2352:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294917837 00000100:00000001:0.0:1773941478.304248:0:2352:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941478.304250:0:2352:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941478.304252:0:2352:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941478.304254:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941478.304494:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941478.304505:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000400:00000200:3.0:1773941478.304514:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941478.304540:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941478.304548:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6cee0 00000400:00000010:3.0:1773941478.304551:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6cee0. 00000100:00000001:3.0:1773941478.304555:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941478.304562:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b850aad80 x1860112241155968/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941494 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941478.304573:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941478.304576:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b850aad80 x1860112241155968/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941494 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941478.304582:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941478.304584:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941478.305155:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941478.305167:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941478.305169:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941478.305171:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941478.305193:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941478.305200:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941478.305208:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941478.305213:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941478.305219:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941478.305224:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321605f80 00000400:00000200:3.0:1773941478.305229:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941478.305236:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xf45 [1] + 224 00000800:00000001:3.0:1773941478.305246:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941478.305272:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941478.305275:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941478.305345:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941478.305350:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941478.305353:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b850aad80 x1860112241155968/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941494 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941478.305365:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b850aad80 x1860112241155968/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941494 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941478.305385:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941478.305394:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941478.305398:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941478.305420:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941478.305423:0:2352:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941478.305428:0:2352:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941478.305429:0:2352:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941478.305432:0:2352:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941478.305437:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941478.305438:0:2352:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941478.305448:0:2352:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941478.305453:0:2352:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b850aad80 x1860112241155968/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941494 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941478.305462:0:2352:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b850aad80 x1860112241155968/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941494 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941478.305467:0:2352:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941478.305468:0:2352:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941478.305470:0:2352:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941478.305472:0:2352:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941478.305474:0:2352:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941478.305475:0:2352:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941478.305477:0:2352:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941478.305482:0:2352:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=57 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941478.305485:0:2352:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941478.305487:0:2352:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941478.305489:0:2352:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941478.305491:0:2352:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941478.305493:0:2352:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941478.305496:0:2352:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941478.305499:0:2352:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941478.305502:0:2352:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941478.305504:0:2352:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941478.305505:0:2352:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941478.305507:0:2352:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941478.305509:0:2352:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b850aad80 x1860112241155968/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941494 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941478.305515:0:2352:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941478.305516:0:2352:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941478.305517:0:2352:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941478.305519:0:2352:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941478.305521:0:2352:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941478.305522:0:2352:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941478.305523:0:2352:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941478.305527:0:2352:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941478.305528:0:2352:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941478.305529:0:2352:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941478.305530:0:2352:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941478.305532:0:2352:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941478.305534:0:2352:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941478.305536:0:2352:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941478.305538:0:2352:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 247 00000400:00000200:0.0:1773941478.305542:0:2352:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941478.305545:0:2352:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941478.305547:0:2352:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941478.305550:0:2352:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941478.305551:0:2352:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941478.305553:0:2352:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:0.0:1773941478.305555:0:2352:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941478.305557:0:2352:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941478.305559:0:2352:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941478.305560:0:2352:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941478.305562:0:2352:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941478.305564:0:2352:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:0.0:1773941478.305565:0:2352:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941478.305566:0:2352:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941478.305568:0:2352:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941478.305569:0:2352:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:0.0:1773941478.305571:0:2352:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:0.0:1773941478.305573:0:2352:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:0.0:1773941478.305574:0:2352:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941478.305576:0:2352:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941478.305578:0:2352:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941478.305580:0:2352:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941478.305582:0:2352:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941478.305583:0:2352:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941478.305590:0:2352:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b850ab480. 00000020:00000040:0.0:1773941478.305593:0:2352:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:0.0:1773941478.305596:0:2352:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941478.305598:0:2352:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941478.305601:0:2352:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941478.305602:0:2352:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941478.305605:0:2352:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941478.305610:0:2352:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b850c0c00. 00000100:00000001:0.0:1773941478.305614:0:2352:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941478.305620:0:2352:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b850ab480 x1860112241156096/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941478.305628:0:2352:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941478.305631:0:2352:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941478.305635:0:2352:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ccd6ef3] to pc [ptlrpcd_rcv+-1] req@ffff8b5b850ab480 x1860112241156096/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941478.305656:0:2352:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941478.305660:0:2352:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941478.305662:0:2352:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:0.0:1773941478.305667:0:2352:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b850aad80 x1860112241155968/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941494 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941478.305675:0:2352:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941478.305679:0:2352:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941478.305682:0:2352:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941478.305687:0:2352:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b850aad80 x1860112241155968/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941494 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941478.305694:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1773941478.305694:0:2352:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1080e6b5-6a6c-4762-b0ec-c785990fec17:2352:1860112241155968:192.168.204.143@tcp:400:kworker.0 00000100:00000001:2.0:1773941478.305699:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941478.305701:0:2352:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941478.305702:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1773941478.305728:0:2352:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941478.305732:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b850ab480 x1860112241156096/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941478.305735:0:2352:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b850aad80 x1860112241155968/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941494 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941478.305755:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941478.305758:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941478.305760:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941478.305763:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941478.305768:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941478.305775:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241156096:192.168.204.143@tcp:250:ptlrpcd_00_00.0 00000100:00000001:0.0:1773941478.305775:0:2352:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941478.305779:0:2352:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941478.305780:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000010:0.0:1773941478.305782:0:2352:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b845c4a00. 02000000:00000001:2.0:1773941478.305784:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941478.305786:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941478.305787:0:2352:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:2.0:1773941478.305788:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1773941478.305791:0:2352:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941478.305793:0:2352:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 02000000:00000010:2.0:1773941478.305794:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce5c00. 00000020:00000001:0.0:1773941478.305796:0:2352:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:2.0:1773941478.305798:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941478.305800:0:2352:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b91e1e300. 00000400:00000200:2.0:1773941478.305804:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 02000000:00000001:0.0:1773941478.305806:0:2352:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000400:00000010:2.0:1773941478.305807:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 02000000:00000001:0.0:1773941478.305808:0:2352:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941478.305813:0:2352:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b850aad80. 00000400:00000010:2.0:1773941478.305815:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000001:0.0:1773941478.305816:0:2352:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941478.305819:0:2352:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1773941478.305820:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241156096, portal 25 00000100:00000001:0.0:1773941478.305821:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941478.305827:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941478.305828:0:2352:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941478.305830:0:2352:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941478.305831:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941478.305833:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941478.305836:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615823923197056 : -128249786354560 : ffff8b5b850ab480) 00000100:00000040:2.0:1773941478.305843:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b850ab480 x1860112241156096/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941488 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941478.305849:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941478.305851:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941478.305853:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34aa0. 00000100:00000200:2.0:1773941478.305855:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241156096, offset 0 00000400:00000200:2.0:1773941478.305858:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941478.305862:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941478.305865:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941478.305867:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941478.305870:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941478.305871:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941478.305873:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941478.305875:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[92, 93] 00000400:00000200:2.0:1773941478.305878:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941478.305880:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [199:199:256:4294967295] 192.168.204.143@tcp LPNI seq info [94:94:8:4294967295] 00000400:00000200:2.0:1773941478.305883:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941478.305887:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941478.305891:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941478.305894:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0300. 00000800:00000200:2.0:1773941478.305897:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941478.305902:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941478.305905:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0300 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941478.305917:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941478.305920:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941478.305922:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294918669 00000100:00000001:2.0:1773941478.305925:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294918667 00000100:00000001:2.0:1773941478.305928:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294918669 00000100:00000001:2.0:1773941478.305930:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294918667 00000100:00000001:2.0:1773941478.305936:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941478.305938:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941478.305940:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941478.305942:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941478.305946:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941478.305947:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941478.305949:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941478.305950:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941478.305952:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941478.305954:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941478.306006:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941478.306011:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0300. 00000400:00000200:3.0:1773941478.306017:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941478.306023:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941478.306028:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34aa0 00000400:00000010:3.0:1773941478.306031:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34aa0. 00000100:00000001:3.0:1773941478.306034:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941478.306040:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b850ab480 x1860112241156096/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941488 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941478.306093:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941478.306099:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b850ab480 x1860112241156096/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941488 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941478.306107:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941478.306109:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941478.307430:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941478.307438:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941478.307440:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941478.307441:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941478.307445:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941478.307449:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941478.307453:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941478.307456:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941478.307458:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941478.307461:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606000 00000400:00000200:3.0:1773941478.307506:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941478.307510:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xf4d [1] + 0 00000800:00000001:3.0:1773941478.307514:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941478.307520:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941478.307521:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941478.307524:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941478.307527:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941478.307530:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b850ab480 x1860112241156096/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941488 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941478.307538:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b850ab480 x1860112241156096/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941488 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941478.307554:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941478.307559:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941478.307562:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941478.307594:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941478.307597:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941478.307600:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941478.307602:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941478.307604:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941478.307608:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941478.307609:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941478.307612:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941478.307617:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b850ab480 x1860112241156096/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941488 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941478.307625:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b850ab480 x1860112241156096/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941488 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941478.307630:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941478.307631:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941478.307633:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941478.307634:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941478.307635:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941478.307637:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941478.307638:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941478.307640:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=57 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941478.307643:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941478.307644:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941478.307645:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941478.307647:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941478.307649:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941478.307651:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941478.307661:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b850ab480 x1860112241156096/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941488 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941478.307666:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941478.307667:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941478.307669:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941478.307671:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941478.307675:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941478.307676:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941478.307677:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941478.307678:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941478.307680:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941478.307681:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941478.307683:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941478.307686:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941478 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941478.307688:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941478.307693:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941478.307695:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941478.307696:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941478.307697:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941478.307698:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941478.307701:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941478.307718:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941478.307720:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941478.307723:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941478.307725:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941478.307727:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941478.308019:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941478.308022:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941478.308026:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b850ab480 x1860112241156096/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941488 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941478.308033:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850ab480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241156096:192.168.204.143@tcp:250:ptlrpcd_00_00.0 00000100:00000001:0.0:1773941478.308042:0:12201:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941478.308045:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941478.308048:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1773941478.308052:0:12201:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000040:2.0:1773941478.308053:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b850ab480 x1860112241156096/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941488 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941478.308057:0:12201:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941478.308061:0:12201:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941478.308064:0:12201:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941478.308066:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941478.308067:0:12201:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941478.308068:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941478.308071:0:12201:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941478.308072:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce5c00. 02000000:00000001:2.0:1773941478.308076:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941478.308077:0:12201:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941478.308078:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941478.308080:0:12201:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000040:2.0:1773941478.308081:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941478.308083:0:12201:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:2.0:1773941478.308084:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:0.0:1773941478.308086:0:12201:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941478.308088:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b850c0c00. 10000000:00000001:0.0:1773941478.308089:0:12201:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941478.308093:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1773941478.308093:0:12201:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:2.0:1773941478.308096:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:0.0:1773941478.308097:0:12201:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000010:2.0:1773941478.308098:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b850ab480. 00000100:00000001:2.0:1773941478.308103:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941478.308103:0:12201:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941478.308106:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941478.308107:0:12201:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941478.308109:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941478.308110:0:12201:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941478.308115:0:12201:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941478.308116:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941478.308120:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941478.308123:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941478.308125:0:12201:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941478.308127:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:0.0:1773941478.308128:0:12201:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941478.308130:0:12201:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941478.308153:0:12201:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941478.308154:0:12201:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941478.308156:0:12201:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941478.308159:0:12201:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941478.308162:0:12201:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941478.308164:0:12201:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941478.308166:0:12201:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941478.308168:0:12201:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941478.308169:0:12201:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941479.263916:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941479.263924:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941479.263929:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941479.327833:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941479.327840:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941479.327846:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941479.327848:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941479.327850:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941479.327852:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941479.327855:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941479.327857:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941479.327860:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941479.327863:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941479.327865:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941479.327876:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90276d80. 00000020:00000040:3.0:1773941479.327883:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941479.327888:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941479.327891:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941479.327896:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941479.327905:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2600. 00000100:00000001:3.0:1773941479.327912:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941479.327917:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941479.327921:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941479.327943:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962500. 00010000:00000001:3.0:1773941479.327949:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941479.327953:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81484a00. 00000020:00000001:3.0:1773941479.327958:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941479.327960:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81484a00 with handle 0xa33a5be14eae1d55 to hash 00000020:00000001:3.0:1773941479.327963:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941479.327965:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00010000:00000001:3.0:1773941479.327968:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00010000:00000001:3.0:1773941479.327972:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941479.327976:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941479.327981:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1d55 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941479.327992:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1d55 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941479.328003:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1d55 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941479.328010:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941479.328013:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941479.328020:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e9c0. 00000100:00000001:3.0:1773941479.328023:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387402176 : -128249322149440 : ffff8b5ba0b5e9c0) 00000100:00000001:3.0:1773941479.328026:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941479.328028:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109628800 : -128249599922816 : ffff8b5b90276d80) 00000100:00000001:3.0:1773941479.328033:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941479.328038:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941479.328040:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941479.328042:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941479.328053:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90276d80 x1860112241156224/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941479.328063:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941479.328065:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941479.328075:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941479.328077:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941479.328080:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941479.328085:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241156224:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941479.328089:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941479.328096:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941479.328098:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941479.328101:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941479.328105:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941479.328108:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941479.328126:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941479.328134:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab210. 00000400:00000010:3.0:1773941479.328141:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a770. 00000100:00000200:3.0:1773941479.328148:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241156224, portal 25 00000100:00000001:3.0:1773941479.328150:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941479.328151:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109628800 : -128249599922816 : ffff8b5b90276d80) 00000100:00000040:3.0:1773941479.328171:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90276d80 x1860112241156224/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941495 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941479.328180:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941479.328183:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941479.328187:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a990. 00000100:00000200:3.0:1773941479.328190:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241156224, offset 0 00000400:00000200:3.0:1773941479.328198:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941479.328207:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941479.328215:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941479.328220:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941479.328224:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941479.328228:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941479.328231:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941479.328234:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[94, 93] 00000400:00000200:3.0:1773941479.328238:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941479.328242:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [200:200:256:4294967295] 192.168.204.153@tcp LPNI seq info [95:95:8:4294967295] 00000400:00000200:3.0:1773941479.328252:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941479.328261:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941479.328265:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941479.328269:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69f00. 00000800:00000200:3.0:1773941479.328275:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941479.328281:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941479.328285:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69f00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941479.328306:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941479.328311:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941479.328313:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294919691 00000100:00000001:3.0:1773941479.328321:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294918669 00000100:00000001:3.0:1773941479.328327:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294919691 00000100:00000001:3.0:1773941479.328332:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294918669 00000100:00000001:3.0:1773941479.328338:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941479.328341:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941479.328343:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941479.328345:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941479.328348:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941479.328351:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941479.328354:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941479.328421:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941479.328426:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69f00. 00000400:00000200:2.0:1773941479.328431:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941479.328436:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941479.328440:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a990 00000400:00000010:2.0:1773941479.328442:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a990. 00000100:00000001:2.0:1773941479.328446:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941479.328450:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90276d80 x1860112241156224/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941495 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941479.328461:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941479.328465:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90276d80 x1860112241156224/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941495 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941479.328487:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941479.328489:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941479.329005:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941479.329017:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941479.329019:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941479.329022:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941479.329026:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941479.329031:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941479.329037:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941479.329041:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941479.329090:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941479.329095:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606080 00000400:00000200:3.0:1773941479.329099:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941479.329111:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xf5d [1] + 224 00000800:00000001:3.0:1773941479.329118:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941479.329126:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941479.329129:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941479.329132:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941479.329136:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941479.329141:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90276d80 x1860112241156224/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941495 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941479.329154:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90276d80 x1860112241156224/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941495 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941479.329185:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941479.329192:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941479.329196:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941479.329266:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941479.329278:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941479.329279:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941479.329283:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab210. 00000400:00000200:1.0:1773941479.329291:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a770 00000400:00000010:1.0:1773941479.329293:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a770. 00000100:00000001:1.0:1773941479.329297:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941479.329306:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90276d80 x1860112241156224/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941495 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941479.329326:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90276d80 x1860112241156224/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941495 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941479.329333:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941479.329335:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941479.329337:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941479.329340:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941479.329343:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941479.329345:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941479.329347:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941479.329353:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=58 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941479.329356:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941479.329359:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941479.329363:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941479.329366:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941479.329368:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941479.329370:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941479.329374:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941479.329378:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941479.329397:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941479.329400:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941479.329401:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941479.329403:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941479.329407:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90276d80 x1860112241156224/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941495 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941479.329412:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941479.329413:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941479.329415:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941479.329419:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941479.329421:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941479.329423:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941479.329424:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941479.329425:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941479.329426:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941479.329427:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941479.329428:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941479.329430:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941479.329432:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941479.329435:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941479.329437:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 248 00000400:00000200:1.0:1773941479.329448:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941479.329453:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941479.329456:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941479.329459:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941479.329461:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941479.329463:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941479.329466:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941479.329468:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941479.329470:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941479.329472:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941479.329474:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941479.329475:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941479.329481:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941479.329483:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941479.329484:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941479.329486:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941479.329488:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941479.329490:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941479.329492:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941479.329494:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941479.329497:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941479.329499:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941479.329501:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941479.329503:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941479.329507:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62a00. 00000020:00000040:1.0:1773941479.329510:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941479.329513:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941479.329514:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941479.329517:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941479.329519:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941479.329522:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941479.329528:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0736800. 00000100:00000001:1.0:1773941479.329532:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941479.329537:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d62a00 x1860112241156352/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941479.329544:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941479.329547:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941479.329550:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000acb4b66b] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d62a00 x1860112241156352/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941479.329570:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941479.329572:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941479.329574:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941479.329576:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941479.329579:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941479.329584:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941479.329586:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941479.329588:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941479.329589:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941479.329593:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90276d80 x1860112241156224/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941495 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941479.329599:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941479.329603:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941479.329605:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941479.329607:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941479.329611:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90276d80 x1860112241156224/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941495 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941479.329616:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1773941479.329617:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241156224:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941479.329624:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941479.329624:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941479.329631:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941479.329631:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941479.329639:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941479.329641:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941479.329647:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90276d80 x1860112241156224/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941495 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941479.329648:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62a00 x1860112241156352/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941479.329664:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941479.329671:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e9c0. 00000100:00000001:1.0:1773941479.329677:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941479.329679:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941479.329686:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:2.0:1773941479.329692:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:1.0:1773941479.329693:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941479.329698:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941479.329699:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941479.329700:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484a00 refcount=4 02000000:00000001:2.0:1773941479.329737:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941479.329740:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941479.329744:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941479.329751:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00000100:00100000:2.0:1773941479.329752:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241156352:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941479.329758:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00000100:00000001:2.0:1773941479.329766:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00010000:1.0:1773941479.329766:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1d55 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941479.329771:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941479.329778:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941479.329789:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941479.329790:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941479.329799:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1d55 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:2.0:1773941479.329800:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce6400. 02000000:00000001:2.0:1773941479.329811:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941479.329815:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941479.329818:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1d55 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941479.329823:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941479.329833:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000010:2.0:1773941479.329834:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00010000:00000001:1.0:1773941479.329835:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941479.329838:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1d55 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941479.329843:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00000001:1.0:1773941479.329843:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941479.329848:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941479.329852:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241156352, portal 25 00010000:00000001:1.0:1773941479.329852:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941479.329855:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1d55 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941479.329859:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00010000:1.0:1773941479.329864:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81484a00 already unused, calling callback (ffffffffc1482c20) 00000100:00000001:2.0:1773941479.329865:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121080320 : -128249588471296 : ffff8b5b90d62a00) 10000000:00000001:1.0:1773941479.329868:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941479.329873:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1d55 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941479.329881:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62a00 x1860112241156352/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941489 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941479.329881:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941479.329885:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941479.329887:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941479.329889:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484a00 refcount=5 00000020:00000001:1.0:1773941479.329895:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00010000:00000001:1.0:1773941479.329902:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860124160 : -128249849427456 : ffff8b5b81484a00) 00000100:00000001:2.0:1773941479.329914:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941479.329915:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941479.329918:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1d55 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941479.329926:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 10000000:00000001:1.0:1773941479.329934:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000010:2.0:1773941479.329938:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34cc0. 10000000:00010000:1.0:1773941479.329939:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1d55 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941479.329949:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241156352, offset 0 10000000:01000000:1.0:1773941479.329951:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941479.329957:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941479.329959:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941479.329963:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941479.329967:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941479.329968:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941479.329972:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941479.329973:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941479.329975:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941479.329976:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941479.329978:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81484a00 with handle 0xa33a5be14eae1d55 from hash 00010000:00000001:1.0:1773941479.329981:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941479.329983:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941479.329984:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941479.329987:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000400:00000200:2.0:1773941479.329989:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941479.329990:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941479.329995:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941479.329999:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941479.330001:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941479.330002:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941479.330007:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941479.330008:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00010000:1.0:1773941479.330010:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1d55 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941479.330019:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941479.330024:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941479.330026:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941479.330028:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000400:00000200:2.0:1773941479.330032:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941479.330033:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000400:00000200:2.0:1773941479.330039:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941479.330040:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941479.330044:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941479.330046:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941479.330049:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:1.0:1773941479.330051:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81484a00/0xa33a5be14eae1d55 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941479.330062:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[94, 95] 00010000:00000040:1.0:1773941479.330066:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962500 count: 0 00000400:00000200:2.0:1773941479.330074:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941479.330082:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [201:201:256:4294967295] 192.168.204.143@tcp LPNI seq info [96:96:8:4294967295] 00010000:00000001:1.0:1773941479.330087:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941479.330092:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941479.330093:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941479.330099:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941479.330107:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90276d80 x1860112241156224/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941495 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941479.330115:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000100:00000001:1.0:1773941479.330132:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941479.330135:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941479.330139:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 00000800:00000200:2.0:1773941479.330141:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 02000000:00000001:1.0:1773941479.330143:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941479.330148:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941479.330150:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000800:00000010:2.0:1773941479.330151:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0500. 00000020:00000001:1.0:1773941479.330153:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941479.330169:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2600. 02000000:00000001:1.0:1773941479.330174:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941479.330176:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000800:00000200:2.0:1773941479.330182:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000100:00000010:1.0:1773941479.330182:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90276d80. 00000100:00000001:1.0:1773941479.330195:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941479.330199:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1773941479.330200:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 10000000:00000001:1.0:1773941479.330205:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000200:2.0:1773941479.330207:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0500 type 1, nob 616 niov 1 nkiov 1 10000000:01000000:1.0:1773941479.330210:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941479.330218:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941479.330222:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000100:00000001:2.0:1773941479.330227:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941479.330227:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000400:00000200:2.0:1773941479.330240:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000010:1.0:1773941479.330240:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000100:00000001:2.0:1773941479.330248:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294919693 00000040:00000010:1.0:1773941479.330251:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000100:00000001:2.0:1773941479.330256:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294919691 00000020:00000010:1.0:1773941479.330257:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000100:00000001:2.0:1773941479.330270:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294919693 00000020:00000010:1.0:1773941479.330271:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941479.330286:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000001:2.0:1773941479.330289:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294919691 00000100:00000001:2.0:1773941479.330293:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941479.330301:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941479.330304:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:1.0:1773941479.330305:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 00000100:00000001:2.0:1773941479.330307:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 20000000:00000010:1.0:1773941479.330314:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 00000100:00000001:2.0:1773941479.330317:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941479.330320:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 10000000:00000040:1.0:1773941479.330321:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000800:00000200:3.0:1773941479.330324:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941479.330324:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000001:1.0:1773941479.330326:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941479.330328:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000800:00000010:3.0:1773941479.330333:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0500. 00000100:00000001:2.0:1773941479.330335:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941479.330338:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000400:00000200:3.0:1773941479.330344:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000040:00000010:1.0:1773941479.330344:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000100:00000001:2.0:1773941479.330345:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941479.330364:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000400:00000200:3.0:1773941479.330370:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000020:00000010:1.0:1773941479.330373:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 00000400:00000200:3.0:1773941479.330378:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34cc0 80000000:00000010:1.0:1773941479.330379:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00000400:00000010:3.0:1773941479.330389:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34cc0. 00040000:00000010:1.0:1773941479.330391:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941479.330400:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000100:00000001:3.0:1773941479.330401:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:1.0:1773941479.330404:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941479.330408:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941479.330412:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000100:00000200:3.0:1773941479.330414:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62a00 x1860112241156352/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941489 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941479.330416:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941479.330425:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941479.330428:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941479.330433:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941479.330436:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941479.330441:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:3.0:1773941479.330497:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941479.330502:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62a00 x1860112241156352/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941489 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941479.330511:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941479.330513:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941479.331575:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941479.331581:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941479.331584:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941479.331585:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941479.331589:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941479.331627:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941479.331632:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941479.331634:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941479.331637:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941479.331639:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606100 00000400:00000200:3.0:1773941479.331642:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941479.331645:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xf6d [1] + 0 00000800:00000001:3.0:1773941479.331649:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941479.331653:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941479.331655:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941479.331664:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941479.331667:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941479.331671:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62a00 x1860112241156352/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941489 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941479.331678:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d62a00 x1860112241156352/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941489 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941479.331691:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941479.331694:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941479.331696:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941479.331736:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941479.331738:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941479.331743:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941479.331744:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941479.331746:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941479.331749:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941479.331750:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941479.331752:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941479.331756:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62a00 x1860112241156352/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941489 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941479.331764:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62a00 x1860112241156352/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941489 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941479.331769:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941479.331770:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941479.331772:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941479.331774:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941479.331775:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941479.331777:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941479.331779:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941479.331784:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=58 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941479.331786:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941479.331788:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941479.331789:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941479.331791:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941479.331793:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941479.331795:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941479.331798:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62a00 x1860112241156352/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941489 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941479.331802:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941479.331804:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941479.331805:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941479.331807:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941479.331819:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941479.331824:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941479.331831:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941479.331832:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941479.331834:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941479.331836:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941479.331838:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941479.331841:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941479 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941479.331843:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941479.331848:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941479.331850:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941479.331852:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941479.331853:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941479.331855:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941479.331857:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941479.331859:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941479.331861:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941479.331864:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941479.331866:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941479.331868:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941479.332053:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941479.332056:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941479.332061:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62a00 x1860112241156352/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941489 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941479.332067:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241156352:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941479.332070:0:12202:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941479.332073:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941479.332076:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941479.332081:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62a00 x1860112241156352/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941489 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941479.332086:0:12202:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:1.0:1773941479.332098:0:12202:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941479.332103:0:12202:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941479.332104:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 10000000:01000000:1.0:1773941479.332106:0:12202:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941479.332108:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941479.332110:0:12202:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941479.332111:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce6400. 00000001:00000001:1.0:1773941479.332115:0:12202:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941479.332117:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941479.332120:0:12202:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941479.332122:0:12202:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941479.332129:0:12202:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:2.0:1773941479.332130:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:1.0:1773941479.332138:0:12202:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941479.332139:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 10000000:00000001:1.0:1773941479.332152:0:12202:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941479.332153:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941479.332178:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0736800. 00000100:00000001:1.0:1773941479.332178:0:12202:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:2.0:1773941479.332184:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941479.332185:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:1.0:1773941479.332186:0:12202:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000010:2.0:1773941479.332188:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62a00. 00000100:00000001:2.0:1773941479.332196:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941479.332196:0:12202:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941479.332198:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941479.332200:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941479.332203:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941479.332204:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941479.332208:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941479.332209:0:12202:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941479.332210:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941479.332212:0:12202:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941479.332217:0:12202:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941479.332219:0:12202:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941479.332222:0:12202:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941479.332224:0:12202:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941479.332243:0:12202:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941479.332244:0:12202:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941479.332245:0:12202:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941479.332250:0:12202:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941479.332253:0:12202:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941479.332256:0:12202:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941479.332258:0:12202:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941479.332260:0:12202:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941479.332262:0:12202:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941479.335736:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962500. 00010000:00000010:1.1:1773941479.335743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81484a00. 00000020:00000001:0.0:1773941480.287878:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941480.287885:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941480.287890:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941480.799795:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941480.799802:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941480.799806:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941480.799808:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941480.799810:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941480.799811:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941480.799813:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941480.799815:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941480.799817:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941480.799819:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941480.799820:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941480.799830:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90274380. 00000020:00000040:3.0:1773941480.799834:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941480.799838:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941480.799857:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941480.799862:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941480.799871:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2a00. 00000100:00000001:3.0:1773941480.799878:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941480.799883:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941480.799886:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941480.799895:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962700. 00010000:00000001:3.0:1773941480.799901:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941480.799905:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485000. 00000020:00000001:3.0:1773941480.799910:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941480.799912:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485000 with handle 0xa33a5be14eae1d5c to hash 00000020:00000001:3.0:1773941480.799915:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941480.799917:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860125696 : -128249849425920 : ffff8b5b81485000) 00010000:00000001:3.0:1773941480.799920:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860125696 : -128249849425920 : ffff8b5b81485000) 00010000:00000001:3.0:1773941480.799924:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941480.799927:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941480.799931:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1d5c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941480.799943:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1d5c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941480.799955:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1d5c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941480.799960:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941480.799962:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941480.799968:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e9c0. 00000100:00000001:3.0:1773941480.799969:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387402176 : -128249322149440 : ffff8b5ba0b5e9c0) 00000100:00000001:3.0:1773941480.799971:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941480.799973:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109618048 : -128249599933568 : ffff8b5b90274380) 00000100:00000001:3.0:1773941480.799977:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941480.799981:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941480.799982:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941480.799989:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941480.799998:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90274380 x1860112241156480/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941480.800004:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941480.800006:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941480.800009:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941480.800011:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941480.800013:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941480.800015:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90274380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241156480:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941480.800019:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941480.800023:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941480.800025:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941480.800027:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941480.800031:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90262000. 02000000:00000001:3.0:1773941480.800033:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941480.800039:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941480.800042:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abc60. 00000400:00000010:3.0:1773941480.800048:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a770. 00000100:00000200:3.0:1773941480.800054:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241156480, portal 25 00000100:00000001:3.0:1773941480.800056:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941480.800057:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109618048 : -128249599933568 : ffff8b5b90274380) 00000100:00000040:3.0:1773941480.800060:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90274380 x1860112241156480/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941496 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941480.800066:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941480.800068:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941480.800069:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a990. 00000100:00000200:3.0:1773941480.800071:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241156480, offset 0 00000400:00000200:3.0:1773941480.800077:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941480.800084:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941480.800090:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941480.800094:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941480.800097:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941480.800100:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941480.800106:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941480.800108:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[96, 95] 00000400:00000200:3.0:1773941480.800111:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941480.800115:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [202:202:256:4294967295] 192.168.204.153@tcp LPNI seq info [97:97:8:4294967295] 00000400:00000200:3.0:1773941480.800118:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941480.800126:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941480.800147:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941480.800152:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69500. 00000800:00000200:3.0:1773941480.800158:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941480.800166:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941480.800171:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69500 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941480.800192:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941480.800197:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941480.800200:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294921163 00000100:00000001:3.0:1773941480.800207:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294919693 00000100:00000001:3.0:1773941480.800213:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294921163 00000100:00000001:3.0:1773941480.800217:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294919693 00000100:00000001:3.0:1773941480.800221:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941480.800225:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941480.800227:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941480.800230:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941480.800232:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941480.800235:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941480.800237:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941480.800299:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941480.800303:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69500. 00000400:00000200:2.0:1773941480.800308:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941480.800313:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941480.800317:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a990 00000400:00000010:2.0:1773941480.800319:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a990. 00000100:00000001:2.0:1773941480.800323:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941480.800327:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90274380 x1860112241156480/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941496 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941480.800335:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941480.800338:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90274380 x1860112241156480/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941496 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941480.800343:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941480.800345:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941480.800964:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941480.800973:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941480.800976:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941480.800977:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941480.800981:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941480.800985:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941480.800989:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941480.800998:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941480.801001:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941480.801004:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606180 00000400:00000200:3.0:1773941480.801007:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941480.801010:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xf7d [1] + 224 00000800:00000001:3.0:1773941480.801069:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941480.801075:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941480.801077:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941480.801080:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941480.801083:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941480.801087:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90274380 x1860112241156480/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941496 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941480.801094:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90274380 x1860112241156480/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941496 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941480.801107:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941480.801113:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941480.801115:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941480.801157:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941480.801163:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941480.801164:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941480.801166:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abc60. 00000400:00000200:1.0:1773941480.801170:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a770 00000400:00000010:1.0:1773941480.801171:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a770. 00000100:00000001:1.0:1773941480.801174:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941480.801178:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90274380 x1860112241156480/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941496 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941480.801186:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90274380 x1860112241156480/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941496 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941480.801192:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941480.801193:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941480.801195:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941480.801197:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941480.801198:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941480.801200:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941480.801202:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941480.801208:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=59 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941480.801212:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941480.801223:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941480.801227:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941480.801230:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941480.801234:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941480.801238:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941480.801242:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941480.801246:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941480.801251:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941480.801254:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941480.801255:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941480.801258:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941480.801262:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90274380 x1860112241156480/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941496 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941480.801271:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941480.801272:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941480.801274:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941480.801277:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941480.801280:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941480.801282:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941480.801284:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941480.801285:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941480.801286:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941480.801288:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941480.801290:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941480.801292:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941480.801294:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941480.801297:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941480.801300:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 249 00000400:00000200:1.0:1773941480.801304:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941480.801307:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941480.801310:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941480.801312:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941480.801319:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941480.801321:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941480.801324:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941480.801325:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941480.801327:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941480.801329:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941480.801331:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941480.801332:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941480.801334:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941480.801335:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941480.801336:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941480.801338:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941480.801340:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941480.801342:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941480.801344:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941480.801346:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941480.801348:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941480.801349:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941480.801351:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941480.801352:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941480.801355:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62680. 00000020:00000040:1.0:1773941480.801357:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941480.801359:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941480.801360:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941480.801362:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941480.801363:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941480.801365:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941480.801368:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0736800. 00000100:00000001:1.0:1773941480.801371:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941480.801376:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d62680 x1860112241156608/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941480.801381:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941480.801387:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941480.801391:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048151604] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d62680 x1860112241156608/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941480.801408:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941480.801410:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941480.801411:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941480.801413:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941480.801415:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941480.801416:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941480.801417:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941480.801419:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941480.801421:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941480.801424:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90274380 x1860112241156480/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941496 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941480.801429:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941480.801431:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941480.801433:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941480.801434:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941480.801438:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90274380 x1860112241156480/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941496 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941480.801443:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90274380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241156480:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941480.801447:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941480.801448:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941480.801451:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941480.801451:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941480.801452:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941480.801457:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90274380 x1860112241156480/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941496 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941480.801458:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941480.801462:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941480.801469:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941480.801472:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62680 x1860112241156608/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941480.801474:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e9c0. 00000100:00000001:1.0:1773941480.801479:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941480.801485:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941480.801488:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941480.801489:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941480.801493:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941480.801496:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941480.801499:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941480.801501:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485000 refcount=4 00000020:00000001:1.0:1773941480.801503:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860125696 : -128249849425920 : ffff8b5b81485000) 02000000:00000001:2.0:1773941480.801511:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941480.801513:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941480.801515:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860125696 : -128249849425920 : ffff8b5b81485000) 02000000:00000001:2.0:1773941480.801517:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941480.801523:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241156608:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00010000:1.0:1773941480.801527:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1d5c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941480.801529:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941480.801533:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941480.801536:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941480.801541:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941480.801545:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000010:2.0:1773941480.801547:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce6800. 00010000:00010000:1.0:1773941480.801551:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1d5c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941480.801552:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941480.801561:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941480.801568:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00010000:00000001:1.0:1773941480.801570:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000010:2.0:1773941480.801575:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00010000:1.0:1773941480.801576:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1d5c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941480.801583:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241156608, portal 25 00010000:00000001:1.0:1773941480.801586:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941480.801588:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941480.801590:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941480.801592:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121079424 : -128249588472192 : ffff8b5b90d62680) 00010000:00010000:1.0:1773941480.801593:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1d5c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941480.801602:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000040:2.0:1773941480.801604:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62680 x1860112241156608/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941490 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941480.801605:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941480.801609:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941480.801612:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1d5c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941480.801625:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485000 already unused, calling callback (ffffffffc1482c20) 00000100:00000001:2.0:1773941480.801630:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 10000000:00000001:1.0:1773941480.801630:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941480.801635:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1d5c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941480.801637:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941480.801641:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34dd0. 00000100:00000200:2.0:1773941480.801647:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241156608, offset 0 00010000:00000001:1.0:1773941480.801649:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941480.801652:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941480.801653:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941480.801656:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485000 refcount=5 00000400:00000200:2.0:1773941480.801658:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000020:00000001:1.0:1773941480.801659:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860125696 : -128249849425920 : ffff8b5b81485000) 00010000:00000001:1.0:1773941480.801663:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860125696 : -128249849425920 : ffff8b5b81485000) 00010000:00000001:1.0:1773941480.801666:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:2.0:1773941480.801668:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941480.801670:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1d5c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941480.801677:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 10000000:00000001:1.0:1773941480.801677:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941480.801681:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1d5c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941480.801684:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:01000000:1.0:1773941480.801689:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941480.801693:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 10000000:00000040:1.0:1773941480.801693:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941480.801696:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941480.801699:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:2.0:1773941480.801700:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941480.801701:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941480.801730:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941480.801731:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941480.801733:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941480.801734:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941480.801736:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485000 with handle 0xa33a5be14eae1d5c from hash 00010000:00000001:1.0:1773941480.801738:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941480.801739:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941480.801740:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941480.801741:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941480.801742:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941480.801745:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:2.0:1773941480.801747:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941480.801748:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941480.801751:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941480.801754:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941480.801756:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[96, 97] 10000000:00000001:1.0:1773941480.801764:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941480.801767:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00010000:1.0:1773941480.801769:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1d5c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941480.801777:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [203:203:256:4294967295] 192.168.204.143@tcp LPNI seq info [98:98:8:4294967295] 00010000:00000001:1.0:1773941480.801778:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941480.801780:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941480.801783:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941480.801784:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941480.801786:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941480.801787:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941480.801787:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941480.801789:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485000/0xa33a5be14eae1d5c lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941480.801796:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962700 count: 0 00000400:00000200:2.0:1773941480.801802:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941480.801808:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941480.801811:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941480.801815:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1773941480.801818:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000100:00000040:1.0:1773941480.801823:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90274380 x1860112241156480/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941496 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1773941480.801831:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000100:00000001:1.0:1773941480.801835:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000800:00000010:2.0:1773941480.801838:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0600. 02000000:00000001:1.0:1773941480.801838:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941480.801842:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90262000. 02000000:00000001:1.0:1773941480.801846:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941480.801849:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000800:00000200:2.0:1773941480.801850:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000020:00000040:1.0:1773941480.801852:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941480.801856:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941480.801858:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2a00. 02000000:00000001:1.0:1773941480.801862:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000800:00000200:2.0:1773941480.801863:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 02000000:00000001:1.0:1773941480.801864:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941480.801867:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90274380. 00000100:00000001:1.0:1773941480.801870:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000800:00000200:2.0:1773941480.801871:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0600 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:1.0:1773941480.801872:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941480.801876:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941480.801879:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941480.801881:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941480.801888:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000100:00000001:2.0:1773941480.801891:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941480.801891:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941480.801899:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000400:00000200:2.0:1773941480.801900:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000040:00000010:1.0:1773941480.801904:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000100:00000001:2.0:1773941480.801905:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294921165 00000020:00000010:1.0:1773941480.801910:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000100:00000001:2.0:1773941480.801913:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294921163 00000020:00000010:1.0:1773941480.801914:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941480.801920:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000001:2.0:1773941480.801922:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294921165 00040000:00000010:1.0:1773941480.801925:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 00000100:00000001:2.0:1773941480.801928:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294921163 20000000:00000010:1.0:1773941480.801931:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 00000100:00000001:2.0:1773941480.801936:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:00000040:1.0:1773941480.801936:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941480.801942:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941480.801943:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941480.801948:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941480.801948:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000100:00000001:2.0:1773941480.801955:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000040:00000010:1.0:1773941480.801955:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000800:00000200:3.0:1773941480.801959:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000020:00000010:1.0:1773941480.801959:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000100:00000001:2.0:1773941480.801962:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000800:00000010:3.0:1773941480.801965:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0600. 00000100:00000001:2.0:1773941480.801966:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000010:1.0:1773941480.801966:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 00000100:00000001:2.0:1773941480.801973:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1773941480.801974:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 80000000:00000010:1.0:1773941480.801974:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00000100:00000001:2.0:1773941480.801978:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941480.801982:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:1.0:1773941480.801983:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941480.801988:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000100:00000001:2.0:1773941480.801989:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941480.801993:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000020:00000010:1.0:1773941480.801993:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941480.802000:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 00000400:00000200:3.0:1773941480.802001:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34dd0 10000000:01000000:1.0:1773941480.802003:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000400:00000010:3.0:1773941480.802004:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34dd0. 10000000:00000001:1.0:1773941480.802007:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941480.802010:0:2347:0:(events.c:34:request_out_callback()) Process entered 10000000:00020000:1.0:1773941480.802010:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941480.802014:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941480.802017:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 00000100:00000200:3.0:1773941480.802018:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62680 x1860112241156608/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941490 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000040:1.0:1773941480.802019:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941480.802024:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:3.0:1773941480.802088:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941480.802094:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62680 x1860112241156608/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941490 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941480.802102:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941480.802104:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941480.803276:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941480.803284:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941480.803287:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941480.803289:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941480.803332:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941480.803338:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941480.803344:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941480.803348:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941480.803351:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941480.803354:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606200 00000400:00000200:3.0:1773941480.803359:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941480.803363:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xf8d [1] + 0 00000800:00000001:3.0:1773941480.803368:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941480.803375:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941480.803377:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941480.803381:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941480.803385:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941480.803390:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62680 x1860112241156608/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941490 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941480.803401:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d62680 x1860112241156608/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941490 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941480.803422:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941480.803427:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941480.803430:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941480.803455:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941480.803457:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941480.803461:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941480.803462:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941480.803464:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941480.803467:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941480.803468:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941480.803470:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941480.803474:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62680 x1860112241156608/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941490 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941480.803491:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62680 x1860112241156608/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941490 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941480.803496:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941480.803497:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941480.803499:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941480.803500:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941480.803501:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941480.803503:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941480.803504:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941480.803516:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=59 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941480.803518:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941480.803522:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941480.803523:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941480.803525:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941480.803527:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941480.803528:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941480.803531:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62680 x1860112241156608/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941490 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941480.803536:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941480.803538:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941480.803539:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941480.803541:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941480.803544:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941480.803545:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941480.803546:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941480.803547:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941480.803549:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941480.803550:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941480.803552:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941480.803555:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941480 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941480.803557:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941480.803561:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941480.803566:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941480.803568:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941480.803569:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941480.803570:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941480.803573:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941480.803574:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941480.803576:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941480.803579:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941480.803581:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941480.803583:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941480.803795:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941480.803800:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941480.803806:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62680 x1860112241156608/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941490 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941480.803810:0:12204:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:1.0:1773941480.803820:0:12204:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:2.0:1773941480.803821:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241156608:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00100000:1.0:1773941480.803824:0:12204:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941480.803826:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941480.803827:0:12204:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941480.803830:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:01000000:1.0:1773941480.803830:0:12204:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941480.803835:0:12204:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000040:2.0:1773941480.803836:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62680 x1860112241156608/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941490 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:1.0:1773941480.803838:0:12204:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941480.803842:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941480.803846:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941480.803848:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce6800. 00000001:00000001:1.0:1773941480.803848:0:12204:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941480.803853:0:12204:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941480.803855:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941480.803864:0:12204:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941480.803865:0:12204:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941480.803868:0:2351:0:(genops.c:1236:class_import_put()) Process entered 10000000:00000001:1.0:1773941480.803869:0:12204:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941480.803871:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941480.803872:0:12204:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:2.0:1773941480.803876:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:1.0:1773941480.803878:0:12204:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:2.0:1773941480.803879:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0736800. 02000000:00000001:2.0:1773941480.803885:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941480.803885:0:12204:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:2.0:1773941480.803888:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941480.803889:0:12204:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000010:2.0:1773941480.803890:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62680. 00000100:00000001:1.0:1773941480.803892:0:12204:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941480.803894:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:1.0:1773941480.803895:0:12204:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941480.803896:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941480.803901:0:12204:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941480.803902:0:12204:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941480.803906:0:12204:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:2.0:1773941480.803907:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941480.803912:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941480.803913:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941480.803915:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941480.803916:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941480.803920:0:12204:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941480.803922:0:12204:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941480.803924:0:12204:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941480.803928:0:12204:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941480.803931:0:12204:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941480.803932:0:12204:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941480.803934:0:12204:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941480.803935:0:12204:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941480.803937:0:12204:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941480.807734:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962700. 00010000:00000010:1.1:1773941480.807743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485000. 00000020:00000001:0.0:1773941481.311881:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941481.311886:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941481.311891:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941482.015807:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941482.015814:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941482.015817:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941482.015819:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941482.015821:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941482.015822:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941482.015824:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941482.015826:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941482.015828:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941482.015830:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941482.015848:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941482.015855:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90276680. 00000020:00000040:3.0:1773941482.015859:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941482.015863:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941482.015866:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941482.015871:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941482.015876:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e3a00. 00000100:00000001:3.0:1773941482.015882:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941482.015885:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941482.015888:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941482.015894:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962300. 00010000:00000001:3.0:1773941482.015899:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941482.015901:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941482.015904:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941482.015907:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1d63 to hash 00000020:00000001:3.0:1773941482.015908:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941482.015909:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941482.015912:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941482.015915:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941482.015917:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941482.015920:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1d63 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941482.015928:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1d63 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941482.015938:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1d63 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941482.015943:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941482.015945:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941482.015951:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941482.015954:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941482.015956:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941482.015963:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109627008 : -128249599924608 : ffff8b5b90276680) 00000100:00000001:3.0:1773941482.015969:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941482.015974:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941482.015976:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941482.015979:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941482.015990:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90276680 x1860112241156736/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941482.016000:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941482.016002:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941482.016006:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941482.016009:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941482.016012:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941482.016017:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241156736:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941482.016021:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941482.016026:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941482.016029:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941482.016031:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941482.016036:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90263000. 02000000:00000001:3.0:1773941482.016039:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941482.016046:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941482.016049:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abaa8. 00000400:00000010:3.0:1773941482.016055:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6aee0. 00000100:00000200:3.0:1773941482.016060:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241156736, portal 25 00000100:00000001:3.0:1773941482.016062:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941482.016063:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109627008 : -128249599924608 : ffff8b5b90276680) 00000100:00000040:3.0:1773941482.016067:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90276680 x1860112241156736/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941498 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941482.016073:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941482.016075:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941482.016076:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a880. 00000100:00000200:3.0:1773941482.016079:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241156736, offset 0 00000400:00000200:3.0:1773941482.016083:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941482.016090:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941482.016101:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941482.016105:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941482.016121:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941482.016124:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941482.016127:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941482.016129:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[98, 97] 00000400:00000200:3.0:1773941482.016132:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941482.016135:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [204:204:256:4294967295] 192.168.204.153@tcp LPNI seq info [99:99:8:4294967295] 00000400:00000200:3.0:1773941482.016138:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941482.016146:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941482.016151:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941482.016155:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69d00. 00000800:00000200:3.0:1773941482.016160:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941482.016166:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941482.016170:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69d00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941482.016186:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941482.016189:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941482.016191:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294922379 00000100:00000001:3.0:1773941482.016197:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294921165 00000100:00000001:3.0:1773941482.016200:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294922379 00000100:00000001:3.0:1773941482.016202:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294921165 00000100:00000001:3.0:1773941482.016205:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941482.016207:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941482.016209:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941482.016211:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941482.016214:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941482.016216:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941482.016219:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941482.016306:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941482.016312:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69d00. 00000400:00000200:2.0:1773941482.016316:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941482.016322:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941482.016327:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a880 00000400:00000010:2.0:1773941482.016331:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a880. 00000100:00000001:2.0:1773941482.016336:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941482.016341:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90276680 x1860112241156736/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941498 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941482.016351:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941482.016354:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90276680 x1860112241156736/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941498 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941482.016361:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941482.016363:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941482.016965:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941482.016975:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941482.016978:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941482.016985:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941482.017019:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941482.017024:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941482.017028:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941482.017031:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941482.017035:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941482.017045:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606280 00000400:00000200:3.0:1773941482.017051:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941482.017054:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xf9d [1] + 224 00000800:00000001:3.0:1773941482.017060:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941482.017066:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941482.017068:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941482.017070:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941482.017074:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941482.017077:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90276680 x1860112241156736/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941498 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941482.017085:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90276680 x1860112241156736/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941498 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941482.017099:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941482.017104:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941482.017106:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941482.017162:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941482.017168:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941482.017169:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941482.017171:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abaa8. 00000400:00000200:1.0:1773941482.017175:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6aee0 00000400:00000010:1.0:1773941482.017176:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6aee0. 00000100:00000001:1.0:1773941482.017179:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941482.017183:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90276680 x1860112241156736/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941498 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941482.017191:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90276680 x1860112241156736/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941498 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941482.017196:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941482.017204:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941482.017207:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941482.017210:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941482.017213:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941482.017215:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941482.017218:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941482.017224:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=61 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941482.017228:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941482.017231:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941482.017235:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941482.017238:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941482.017241:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941482.017245:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941482.017249:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941482.017253:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941482.017257:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941482.017259:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941482.017261:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941482.017263:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941482.017267:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90276680 x1860112241156736/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941498 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941482.017276:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941482.017277:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941482.017279:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941482.017282:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941482.017285:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941482.017286:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941482.017288:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941482.017290:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941482.017291:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941482.017293:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941482.017294:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941482.017297:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941482.017298:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941482.017314:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941482.017317:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 251 00000400:00000200:1.0:1773941482.017321:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941482.017327:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941482.017331:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941482.017334:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941482.017337:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941482.017339:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941482.017342:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941482.017343:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941482.017345:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941482.017346:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941482.017348:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941482.017349:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941482.017352:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941482.017353:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941482.017354:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941482.017355:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941482.017357:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941482.017360:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941482.017361:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941482.017363:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941482.017365:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941482.017367:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941482.017368:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941482.017370:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941482.017373:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62d80. 00000020:00000040:1.0:1773941482.017375:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941482.017377:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941482.017378:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941482.017380:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941482.017381:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941482.017386:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941482.017390:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734000. 00000100:00000001:1.0:1773941482.017394:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941482.017400:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d62d80 x1860112241156864/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941482.017409:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941482.017412:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941482.017418:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002abf64a5] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d62d80 x1860112241156864/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941482.017440:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941482.017443:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941482.017445:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941482.017448:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941482.017450:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941482.017452:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941482.017455:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941482.017457:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941482.017460:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941482.017467:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90276680 x1860112241156736/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941498 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941482.017470:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941482.017476:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941482.017479:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941482.017481:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941482.017487:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941482.017489:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62d80 x1860112241156864/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941482.017490:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941482.017495:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941482.017504:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1773941482.017505:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90276680 x1860112241156736/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941498 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941482.017508:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941482.017515:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:1.0:1773941482.017519:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241156736:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941482.017524:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941482.017525:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941482.017528:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941482.017528:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:2.0:1773941482.017536:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941482.017536:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941482.017542:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90276680 x1860112241156736/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941498 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941482.017546:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241156864:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941482.017552:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941482.017556:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:1.0:1773941482.017557:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 02000000:00000001:2.0:1773941482.017561:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941482.017562:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:2.0:1773941482.017566:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941482.017566:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941482.017573:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941482.017573:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941482.017576:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941482.017577:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941482.017579:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 00000020:00000001:1.0:1773941482.017580:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:1.0:1773941482.017584:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 02000000:00000010:2.0:1773941482.017587:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce7000. 00010000:00010000:1.0:1773941482.017588:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1d63 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941482.017593:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941482.017600:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:2.0:1773941482.017604:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941482.017607:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1d63 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941482.017609:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941482.017617:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00000001:1.0:1773941482.017617:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941482.017621:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1d63 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941482.017623:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241156864, portal 25 00000100:00000001:2.0:1773941482.017630:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941482.017631:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941482.017633:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121081216 : -128249588470400 : ffff8b5b90d62d80) 00010000:00000001:1.0:1773941482.017635:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941482.017640:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1d63 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941482.017644:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62d80 x1860112241156864/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941482.017660:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:2.0:1773941482.017663:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941482.017664:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941482.017670:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941482.017672:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000010:2.0:1773941482.017674:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34e58. 00000100:00000200:2.0:1773941482.017677:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241156864, offset 0 00000400:00000200:2.0:1773941482.017683:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941482.017684:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1d63 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941482.017689:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941482.017696:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941482.017729:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941482.017733:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941482.017736:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941482.017739:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941482.017743:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[98, 99] 00010000:00010000:1.0:1773941482.017743:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941482.017749:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941482.017751:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 10000000:00010000:1.0:1773941482.017755:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1d63 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941482.017757:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [205:205:256:4294967295] 192.168.204.143@tcp LPNI seq info [100:100:8:4294967295] 00010000:00000001:1.0:1773941482.017770:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:2.0:1773941482.017771:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941482.017774:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941482.017777:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941482.017779:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000020:00000001:1.0:1773941482.017783:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:1.0:1773941482.017788:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000200:2.0:1773941482.017789:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941482.017791:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941482.017796:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1d63 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941482.017801:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 10000000:00000001:1.0:1773941482.017808:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000010:2.0:1773941482.017810:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0e00. 10000000:00010000:1.0:1773941482.017811:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1d63 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941482.017819:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000800:00000200:2.0:1773941482.017821:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00000040:1.0:1773941482.017824:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941482.017826:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941482.017829:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941482.017832:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000800:00000200:2.0:1773941482.017833:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941482.017835:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000800:00000200:2.0:1773941482.017838:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0e00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941482.017838:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941482.017840:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941482.017842:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941482.017844:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1d63 from hash 00010000:00000001:1.0:1773941482.017847:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941482.017858:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941482.017859:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941482.017860:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:2.0:1773941482.017862:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941482.017862:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941482.017865:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941482.017868:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941482.017870:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941482.017871:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941482.017872:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941482.017877:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294922381 10000000:00000001:1.0:1773941482.017877:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941482.017882:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1d63 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941482.017886:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294922379 00010000:00000001:1.0:1773941482.017888:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941482.017892:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941482.017894:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:2.0:1773941482.017895:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294922381 00010000:00000001:1.0:1773941482.017898:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:2.0:1773941482.017902:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294922379 00010000:00000001:1.0:1773941482.017903:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941482.017907:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941482.017910:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941482.017911:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941482.017916:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941482.017916:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1d63 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941482.017921:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941482.017928:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941482.017929:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:1.0:1773941482.017933:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962300 count: 0 00000800:00000010:3.0:1773941482.017934:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0e00. 00000100:00000001:2.0:1773941482.017938:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941482.017941:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:3.0:1773941482.017942:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941482.017945:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941482.017949:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941482.017951:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941482.017952:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941482.017953:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941482.017956:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:3.0:1773941482.017959:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34e58 00000100:00000001:2.0:1773941482.017959:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941482.017961:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000010:3.0:1773941482.017964:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34e58. 00000100:00000001:3.0:1773941482.017968:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1773941482.017969:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90276680 x1860112241156736/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941498 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941482.017978:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941482.017981:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941482.017983:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90263000. 02000000:00000001:1.0:1773941482.017986:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941482.017987:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941482.017989:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941482.017990:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941482.017992:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e3a00. 02000000:00000001:1.0:1773941482.018000:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941482.018001:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941482.018003:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90276680. 00000100:00000001:1.0:1773941482.018005:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941482.018006:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941482.018008:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941482.018010:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941482.018012:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000200:3.0:1773941482.018015:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62d80 x1860112241156864/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000010:1.0:1773941482.018015:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941482.018021:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941482.018027:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000100:00000001:3.0:1773941482.018029:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000040:00000010:1.0:1773941482.018032:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000100:00000040:3.0:1773941482.018036:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62d80 x1860112241156864/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941492 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941482.018038:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941482.018041:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941482.018046:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941482.018052:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 00000100:00000001:3.0:1773941482.018053:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941482.018057:0:2347:0:(events.c:68:request_out_callback()) Process leaving 20000000:00000010:1.0:1773941482.018057:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 10000000:00000040:1.0:1773941482.018062:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941482.018064:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941482.018068:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941482.018072:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941482.018075:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941482.018078:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941482.018080:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941482.018084:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941482.018086:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941482.018088:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941482.018089:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941482.018091:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941482.018093:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941482.018094:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941482.018096:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941482.018096:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941482.018098:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941482.018103:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941482.019166:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941482.019171:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941482.019173:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941482.019175:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941482.019178:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941482.019181:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941482.019184:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941482.019186:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941482.019189:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941482.019191:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606300 00000400:00000200:3.0:1773941482.019194:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941482.019197:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xfad [1] + 0 00000800:00000001:3.0:1773941482.019200:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941482.019203:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941482.019205:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941482.019208:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941482.019267:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941482.019273:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62d80 x1860112241156864/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941492 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941482.019284:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d62d80 x1860112241156864/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941492 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941482.019301:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941482.019307:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941482.019311:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941482.019334:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941482.019336:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941482.019339:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941482.019340:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941482.019342:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941482.019345:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941482.019346:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941482.019348:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941482.019352:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62d80 x1860112241156864/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941492 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941482.019359:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62d80 x1860112241156864/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941492 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941482.019364:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941482.019365:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941482.019367:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941482.019368:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941482.019369:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941482.019370:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941482.019378:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941482.019380:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=61 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941482.019383:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941482.019384:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941482.019385:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941482.019387:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941482.019389:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941482.019391:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941482.019394:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62d80 x1860112241156864/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941492 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941482.019398:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941482.019400:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941482.019401:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941482.019403:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941482.019406:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941482.019407:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941482.019408:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941482.019410:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941482.019412:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941482.019413:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941482.019415:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941482.019417:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941482 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941482.019419:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941482.019424:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941482.019425:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941482.019426:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941482.019428:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941482.019429:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941482.019431:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941482.019433:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941482.019434:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941482.019437:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941482.019439:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941482.019445:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941482.019633:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941482.019636:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941482.019641:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62d80 x1860112241156864/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941492 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941482.019648:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241156864:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941482.019651:0:12205:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941482.019652:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941482.019656:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941482.019658:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62d80 x1860112241156864/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941492 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941482.019662:0:12205:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:1.0:1773941482.019669:0:12205:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941482.019672:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1773941482.019673:0:12205:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941482.019677:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 10000000:01000000:1.0:1773941482.019677:0:12205:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000010:2.0:1773941482.019680:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce7000. 00000001:00000001:1.0:1773941482.019680:0:12205:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941482.019683:0:12205:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941482.019684:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941482.019688:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:1.0:1773941482.019688:0:12205:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941482.019691:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:1.0:1773941482.019692:0:12205:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941482.019696:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941482.019697:0:12205:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941482.019699:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734000. 00000001:00000001:1.0:1773941482.019701:0:12205:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941482.019723:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941482.019724:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941482.019725:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62d80. 00000100:00000001:2.0:1773941482.019728:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941482.019731:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941482.019733:0:12205:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941482.019734:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941482.019740:0:12205:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941482.019742:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941482.019745:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941482.019746:0:12205:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941482.019747:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941482.019751:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941482.019753:0:12205:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941482.019757:0:12205:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941482.019760:0:12205:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941482.019764:0:12205:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941482.019766:0:12205:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941482.019767:0:12205:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941482.019769:0:12205:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941482.019786:0:12205:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941482.019788:0:12205:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941482.019791:0:12205:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941482.019795:0:12205:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941482.019800:0:12205:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941482.019803:0:12205:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941482.019805:0:12205:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941482.019808:0:12205:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941482.019810:0:12205:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941482.023736:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962300. 00010000:00000010:1.1:1773941482.023744:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:0.0:1773941482.335884:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941482.335889:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941482.335892:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1773941483.359935:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941483.359946:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941483.359953:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1773941483.423794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1773941483.423806:0:37:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1773941483.423819:0:37:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021d500. 00000020:00000040:2.0:1773941483.423825:0:37:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941483.423830:0:37:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941483.423852:0:37:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941483.423858:0:37:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941483.423866:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b98bb0800. 00000100:00000001:2.0:1773941483.423874:0:37:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941483.423886:0:37:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS req@ffff8b5b9021d500 x1860112241156992/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941483.423896:0:37:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941483.423901:0:37:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941483.423907:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1896fa] to pc [ptlrpcd_01_00+0] req@ffff8b5b9021d500 x1860112241156992/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941483.423941:0:37:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941483.423947:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (258) 00000100:00000001:2.0:1773941483.423961:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1773941483.423966:0:2355:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:2.0:1773941483.423969:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941483.423978:0:2355:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941483.423981:0:2355:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941483.423982:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941483.423983:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941483.423985:0:2355:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941483.423986:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.423988:0:2355:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941483.423995:0:2355:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021d500 x1860112241156992/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941483.424004:0:2355:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941483.424007:0:2355:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941483.424011:0:2355:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941483.424014:0:2355:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941483.424018:0:2355:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941483.424022:0:2355:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2355:1860112241156992:192.168.204.143@tcp:400:kworker.0 00000100:00000001:2.0:1773941483.424026:0:2355:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941483.424032:0:2355:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941483.424035:0:2355:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941483.424037:0:2355:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941483.424042:0:2355:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b90e7f400. 02000000:00000001:2.0:1773941483.424043:0:2355:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941483.424057:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941483.424061:0:2355:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941483.424068:0:2355:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000200:2.0:1773941483.424076:0:2355:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241156992, portal 25 00000100:00000001:2.0:1773941483.424078:0:2355:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941483.424102:0:2355:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109262080 : -128249600289536 : ffff8b5b9021d500) 00000100:00000040:2.0:1773941483.424109:0:2355:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021d500 x1860112241156992/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941499 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941483.424115:0:2355:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941483.424118:0:2355:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941483.424120:0:2355:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f346e8. 00000100:00000200:2.0:1773941483.424123:0:2355:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241156992, offset 0 00000400:00000200:2.0:1773941483.424129:0:2355:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941483.424138:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941483.424146:0:2355:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941483.424152:0:2355:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941483.424156:0:2355:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941483.424159:0:2355:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941483.424162:0:2355:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941483.424164:0:2355:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[100, 99] 00000400:00000200:2.0:1773941483.424167:0:2355:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:2.0:1773941483.424170:0:2355:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [206:206:256:4294967295] 192.168.204.153@tcp LPNI seq info [101:101:8:4294967295] 00000400:00000200:2.0:1773941483.424174:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941483.424182:0:2355:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:2.0:1773941483.424188:0:2355:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:2.0:1773941483.424191:0:2355:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0200. 00000800:00000200:2.0:1773941483.424198:0:2355:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:2.0:1773941483.424206:0:2355:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:2.0:1773941483.424210:0:2355:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0200 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1773941483.424224:0:2355:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941483.424229:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941483.424238:0:2355:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294923787 00000100:00000001:2.0:1773941483.424247:0:2355:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294922381 00000100:00000001:2.0:1773941483.424251:0:2355:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294923787 00000100:00000001:2.0:1773941483.424254:0:2355:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294922381 00000100:00000001:2.0:1773941483.424258:0:2355:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941483.424261:0:2355:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.424265:0:2355:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.424267:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941483.424370:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941483.424374:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0200. 00000400:00000200:3.0:1773941483.424381:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941483.424386:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941483.424396:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f346e8 00000400:00000010:3.0:1773941483.424399:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f346e8. 00000100:00000001:3.0:1773941483.424402:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941483.424406:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021d500 x1860112241156992/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941499 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941483.424414:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941483.424416:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021d500 x1860112241156992/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941499 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941483.424422:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941483.424423:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941483.425012:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.425024:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941483.425027:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.425029:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941483.425033:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941483.425037:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941483.425042:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941483.425045:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941483.425095:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941483.425101:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606380 00000400:00000200:3.0:1773941483.425105:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941483.425111:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xfbd [1] + 224 00000800:00000001:3.0:1773941483.425120:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.425129:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941483.425132:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941483.425135:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941483.425140:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941483.425145:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021d500 x1860112241156992/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941499 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941483.425154:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b9021d500 x1860112241156992/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941499 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941483.425168:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941483.425182:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.425186:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941483.425197:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941483.425199:0:2355:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941483.425205:0:2355:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941483.425207:0:2355:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941483.425210:0:2355:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941483.425213:0:2355:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941483.425215:0:2355:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941483.425218:0:2355:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941483.425221:0:2355:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021d500 x1860112241156992/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941499 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941483.425228:0:2355:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021d500 x1860112241156992/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941499 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941483.425233:0:2355:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941483.425234:0:2355:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941483.425236:0:2355:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941483.425238:0:2355:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941483.425240:0:2355:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941483.425242:0:2355:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941483.425244:0:2355:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941483.425251:0:2355:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=62 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941483.425254:0:2355:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941483.425256:0:2355:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941483.425258:0:2355:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941483.425261:0:2355:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941483.425263:0:2355:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941483.425267:0:2355:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941483.425276:0:2355:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941483.425278:0:2355:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941483.425281:0:2355:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941483.425282:0:2355:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941483.425284:0:2355:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941483.425287:0:2355:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b9021d500 x1860112241156992/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941499 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941483.425292:0:2355:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941483.425293:0:2355:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941483.425295:0:2355:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941483.425298:0:2355:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941483.425300:0:2355:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941483.425302:0:2355:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.425303:0:2355:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941483.425304:0:2355:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941483.425305:0:2355:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941483.425306:0:2355:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.425308:0:2355:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941483.425310:0:2355:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941483.425312:0:2355:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941483.425314:0:2355:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941483.425316:0:2355:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 252 00000400:00000200:2.0:1773941483.425319:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941483.425322:0:2355:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941483.425325:0:2355:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941483.425328:0:2355:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941483.425330:0:2355:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941483.425331:0:2355:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:2.0:1773941483.425334:0:2355:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941483.425335:0:2355:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941483.425338:0:2355:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941483.425340:0:2355:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941483.425342:0:2355:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941483.425348:0:2355:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:2.0:1773941483.425351:0:2355:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941483.425353:0:2355:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941483.425354:0:2355:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941483.425356:0:2355:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:2.0:1773941483.425358:0:2355:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:2.0:1773941483.425360:0:2355:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:2.0:1773941483.425362:0:2355:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941483.425364:0:2355:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941483.425366:0:2355:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941483.425368:0:2355:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.425370:0:2355:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941483.425371:0:2355:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941483.425374:0:2355:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021ca80. 00000020:00000040:2.0:1773941483.425375:0:2355:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:2.0:1773941483.425378:0:2355:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941483.425379:0:2355:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.425382:0:2355:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941483.425383:0:2355:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941483.425385:0:2355:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941483.425389:0:2355:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0ce7000. 00000100:00000001:2.0:1773941483.425391:0:2355:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941483.425396:0:2355:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b9021ca80 x1860112241157120/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941483.425401:0:2355:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941483.425403:0:2355:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941483.425406:0:2355:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000005f3b654] to pc [ptlrpcd_rcv+-1] req@ffff8b5b9021ca80 x1860112241157120/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941483.425426:0:2355:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.425428:0:2355:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941483.425429:0:2355:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:2.0:1773941483.425433:0:2355:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021d500 x1860112241156992/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941499 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941483.425443:0:2355:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941483.425446:0:2355:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941483.425448:0:2355:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941483.425453:0:2355:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021d500 x1860112241156992/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941499 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941483.425453:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941483.425458:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941483.425462:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1773941483.425465:0:2355:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2355:1860112241156992:192.168.204.143@tcp:400:kworker.0 00000100:00000001:2.0:1773941483.425472:0:2355:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941483.425472:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021ca80 x1860112241157120/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941483.425477:0:2355:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941483.425483:0:2355:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021d500 x1860112241156992/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941499 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941483.425491:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941483.425496:0:2355:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941483.425496:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941483.425502:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941483.425503:0:2355:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941483.425510:0:2355:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b90e7f400. 02000000:00000001:2.0:1773941483.425513:0:2355:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:0.0:1773941483.425515:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1773941483.425516:0:2355:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941483.425519:0:2355:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 02000000:00000001:0.0:1773941483.425522:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941483.425523:0:2355:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941483.425526:0:2355:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b98bb0800. 02000000:00000001:2.0:1773941483.425530:0:2355:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00100000:0.0:1773941483.425532:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241157120:192.168.204.143@tcp:250:ptlrpcd_01_01.0 02000000:00000001:2.0:1773941483.425533:0:2355:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941483.425535:0:2355:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021d500. 00000100:00000001:2.0:1773941483.425539:0:2355:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941483.425541:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1773941483.425542:0:2355:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941483.425546:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1773941483.425548:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1773941483.425549:0:2355:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941483.425552:0:2355:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941483.425552:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.425557:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1773941483.425559:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1773941483.425560:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941483.425567:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0000. 02000000:00000001:0.0:1773941483.425579:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941483.425583:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941483.425587:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941483.425590:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941483.425593:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241157120, portal 25 00000100:00000001:0.0:1773941483.425595:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941483.425596:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109259392 : -128249600292224 : ffff8b5b9021ca80) 00000100:00000040:0.0:1773941483.425601:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021ca80 x1860112241157120/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941483.425607:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941483.425609:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941483.425611:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c770. 00000100:00000200:0.0:1773941483.425614:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241157120, offset 0 00000400:00000200:0.0:1773941483.425617:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941483.425620:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941483.425623:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941483.425626:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941483.425628:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941483.425630:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941483.425632:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941483.425634:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[100, 101] 00000400:00000200:0.0:1773941483.425637:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941483.425639:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [207:207:256:4294967295] 192.168.204.143@tcp LPNI seq info [102:102:8:4294967295] 00000400:00000200:0.0:1773941483.425645:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941483.425656:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941483.425660:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941483.425663:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00000800:00000200:0.0:1773941483.425668:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941483.425677:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941483.425680:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:0.0:1773941483.425693:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941483.425697:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941483.425700:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294923788 00000100:00000001:0.0:1773941483.425728:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294923787 00000100:00000001:0.0:1773941483.425734:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294923789 00000100:00000001:0.0:1773941483.425738:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294923787 00000100:00000001:0.0:1773941483.425741:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941483.425744:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941483.425748:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941483.425750:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941483.425756:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941483.425758:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1773941483.425760:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941483.425761:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941483.425763:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941483.425765:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941483.425769:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941483.425772:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000400:00000200:3.0:1773941483.425776:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941483.425782:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941483.425785:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c770 00000400:00000010:3.0:1773941483.425787:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c770. 00000100:00000001:3.0:1773941483.425789:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941483.425793:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021ca80 x1860112241157120/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941483.425798:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941483.425801:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021ca80 x1860112241157120/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941493 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941483.425805:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941483.425806:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941483.427146:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.427152:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941483.427153:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.427155:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941483.427158:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941483.427162:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941483.427166:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941483.427168:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941483.427171:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941483.427173:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606400 00000400:00000200:3.0:1773941483.427176:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941483.427179:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xfcd [1] + 0 00000800:00000001:3.0:1773941483.427215:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.427221:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941483.427223:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941483.427225:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941483.427228:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941483.427238:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021ca80 x1860112241157120/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941493 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941483.427247:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b9021ca80 x1860112241157120/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941493 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941483.427260:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941483.427264:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.427266:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941483.427293:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941483.427296:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941483.427300:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941483.427302:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941483.427304:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941483.427308:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941483.427310:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941483.427313:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941483.427318:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021ca80 x1860112241157120/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941493 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941483.427325:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021ca80 x1860112241157120/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941493 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941483.427337:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941483.427338:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941483.427340:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941483.427341:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941483.427342:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941483.427344:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941483.427346:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941483.427348:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=62 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941483.427351:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941483.427352:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941483.427354:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941483.427356:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941483.427358:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941483.427360:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941483.427363:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021ca80 x1860112241157120/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941493 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941483.427367:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941483.427369:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941483.427370:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941483.427372:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941483.427378:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941483.427379:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941483.427381:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941483.427382:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941483.427384:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941483.427385:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941483.427387:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941483.427390:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941483 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941483.427392:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941483.427397:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941483.427399:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941483.427400:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941483.427401:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941483.427406:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941483.427409:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941483.427411:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941483.427413:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941483.427416:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941483.427418:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941483.427420:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941483.427669:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.427675:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941483.427682:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021ca80 x1860112241157120/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941493 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941483.427688:0:12206:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:0.0:1773941483.427700:0:12206:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941483.427720:0:12206:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:2.0:1773941483.427730:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241157120:192.168.204.143@tcp:250:ptlrpcd_01_01.0 00000100:00000001:2.0:1773941483.427739:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941483.427741:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941483.427744:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021ca80 x1860112241157120/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941493 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941483.427748:0:12206:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941483.427749:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 10000000:01000000:0.0:1773941483.427752:0:12206:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941483.427753:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941483.427757:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0000. 00000001:00000001:0.0:1773941483.427758:0:12206:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941483.427760:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941483.427762:0:12206:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:2.0:1773941483.427763:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941483.427767:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941483.427769:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:0.0:1773941483.427769:0:12206:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941483.427773:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0ce7000. 00000001:00000001:0.0:1773941483.427773:0:12206:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941483.427777:0:12206:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941483.427778:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:0.0:1773941483.427779:0:12206:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941483.427780:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:00000001:0.0:1773941483.427782:0:12206:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941483.427783:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021ca80. 00000100:00000001:0.0:1773941483.427785:0:12206:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941483.427788:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941483.427790:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941483.427790:0:12206:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941483.427793:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941483.427794:0:12206:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941483.427797:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941483.427798:0:12206:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941483.427801:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941483.427802:0:12206:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.427806:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1773941483.427807:0:12206:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941483.427809:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941483.427810:0:12206:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941483.427813:0:12206:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941483.427814:0:12206:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941483.427830:0:12206:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941483.427831:0:12206:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941483.427832:0:12206:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941483.427835:0:12206:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941483.427837:0:12206:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941483.427839:0:12206:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941483.427847:0:12206:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941483.427849:0:12206:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941483.427850:0:12206:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941483.735804:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941483.735816:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941483.735823:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941483.735826:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941483.735828:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941483.735830:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941483.735833:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941483.735836:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941483.735838:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941483.735842:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941483.735845:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941483.735860:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277800. 00000020:00000040:3.0:1773941483.735866:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941483.735872:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941483.735877:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941483.735885:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941483.735895:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e3000. 00000100:00000001:3.0:1773941483.735904:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941483.735910:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941483.735914:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941483.735923:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962200. 00010000:00000001:3.0:1773941483.735931:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941483.735935:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81484600. 00000020:00000001:3.0:1773941483.735941:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941483.735944:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81484600 with handle 0xa33a5be14eae1d6a to hash 00000020:00000001:3.0:1773941483.735947:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941483.735949:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:3.0:1773941483.735953:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:3.0:1773941483.735975:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941483.735979:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941483.735984:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d6a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941483.735997:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d6a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941483.736009:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d6a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941483.736014:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941483.736016:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941483.736023:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e300. 00000100:00000001:3.0:1773941483.736025:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387400448 : -128249322151168 : ffff8b5ba0b5e300) 00000100:00000001:3.0:1773941483.736028:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941483.736029:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109631488 : -128249599920128 : ffff8b5b90277800) 00000100:00000001:3.0:1773941483.736034:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941483.736039:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941483.736041:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941483.736042:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941483.736054:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90277800 x1860112241157248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941483.736061:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941483.736063:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941483.736067:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941483.736069:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941483.736071:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941483.736087:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241157248:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941483.736091:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941483.736097:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941483.736100:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941483.736102:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941483.736110:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90262000. 02000000:00000001:3.0:1773941483.736112:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941483.736120:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941483.736124:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845abe18. 00000400:00000010:3.0:1773941483.736134:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a440. 00000100:00000200:3.0:1773941483.736142:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241157248, portal 25 00000100:00000001:3.0:1773941483.736145:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941483.736147:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109631488 : -128249599920128 : ffff8b5b90277800) 00000100:00000040:3.0:1773941483.736154:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90277800 x1860112241157248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941499 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941483.736162:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941483.736166:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941483.736170:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6aa18. 00000100:00000200:3.0:1773941483.736173:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241157248, offset 0 00000400:00000200:3.0:1773941483.736180:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941483.736189:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941483.736198:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941483.736203:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941483.736207:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941483.736211:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941483.736213:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941483.736216:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[102, 101] 00000400:00000200:3.0:1773941483.736218:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941483.736222:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [208:208:256:4294967295] 192.168.204.153@tcp LPNI seq info [103:103:8:4294967295] 00000400:00000200:3.0:1773941483.736226:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941483.736243:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941483.736248:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941483.736254:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69d00. 00000800:00000200:3.0:1773941483.736261:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941483.736268:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941483.736273:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69d00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941483.736299:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941483.736303:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941483.736305:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294924099 00000100:00000001:3.0:1773941483.736312:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294923788 00000100:00000001:3.0:1773941483.736317:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294924099 00000100:00000001:3.0:1773941483.736321:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294923789 00000100:00000001:3.0:1773941483.736323:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941483.736326:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941483.736328:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941483.736330:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941483.736332:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 11 seconds 00000100:00000001:3.0:1773941483.736335:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941483.736338:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941483.736454:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941483.736462:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69d00. 00000400:00000200:2.0:1773941483.736469:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941483.736477:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941483.736485:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6aa18 00000400:00000010:2.0:1773941483.736488:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6aa18. 00000100:00000001:2.0:1773941483.736494:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941483.736501:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277800 x1860112241157248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941499 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941483.736514:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941483.736525:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90277800 x1860112241157248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941499 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941483.736534:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.736537:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941483.736968:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.736982:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941483.736986:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.736989:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941483.736994:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941483.737037:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941483.737044:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941483.737047:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941483.737051:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941483.737055:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606480 00000400:00000200:3.0:1773941483.737058:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941483.737062:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xfd5 [1] + 224 00000800:00000001:3.0:1773941483.737068:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.737086:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941483.737088:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941483.737090:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941483.737095:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941483.737098:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277800 x1860112241157248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941499 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941483.737119:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90277800 x1860112241157248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941499 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941483.737135:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941483.737144:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.737148:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941483.737178:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941483.737185:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941483.737187:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941483.737190:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abe18. 00000400:00000200:1.0:1773941483.737194:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a440 00000400:00000010:1.0:1773941483.737201:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a440. 00000100:00000001:1.0:1773941483.737204:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941483.737209:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277800 x1860112241157248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941499 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941483.737217:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277800 x1860112241157248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941499 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941483.737223:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941483.737225:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941483.737227:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941483.737229:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941483.737232:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941483.737234:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941483.737236:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941483.737242:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=62 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941483.737245:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941483.737247:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941483.737251:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941483.737261:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941483.737264:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941483.737266:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941483.737271:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941483.737276:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941483.737280:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941483.737283:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941483.737285:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941483.737287:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941483.737292:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90277800 x1860112241157248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941499 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941483.737301:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941483.737303:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941483.737305:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941483.737315:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941483.737318:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941483.737320:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941483.737322:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941483.737323:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941483.737324:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941483.737326:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941483.737328:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941483.737330:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941483.737333:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941483.737336:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941483.737340:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 253 00000400:00000200:1.0:1773941483.737345:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941483.737350:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941483.737355:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941483.737359:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941483.737362:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941483.737365:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941483.737368:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941483.737370:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941483.737371:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941483.737374:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941483.737375:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941483.737377:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941483.737379:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941483.737380:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941483.737382:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941483.737383:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941483.737385:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941483.737388:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941483.737391:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941483.737393:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941483.737396:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941483.737402:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941483.737405:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941483.737407:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941483.737410:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63100. 00000020:00000040:1.0:1773941483.737412:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941483.737415:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941483.737416:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941483.737418:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941483.737421:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941483.737423:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941483.737429:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737000. 00000100:00000001:1.0:1773941483.737433:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941483.737440:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63100 x1860112241157376/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941483.737449:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941483.737452:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941483.737457:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000025727364] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63100 x1860112241157376/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941483.737481:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941483.737483:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941483.737485:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941483.737487:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941483.737489:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941483.737490:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941483.737492:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 1 seconds 00000100:00000001:1.0:1773941483.737493:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941483.737495:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941483.737499:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90277800 x1860112241157248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941499 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941483.737505:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941483.737507:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941483.737509:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941483.737511:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941483.737517:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1773941483.737520:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90277800 x1860112241157248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941499 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941483.737523:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941483.737527:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:1.0:1773941483.737533:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241157248:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000040:2.0:1773941483.737537:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63100 x1860112241157376/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941483.737539:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941483.737543:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941483.737548:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941483.737551:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941483.737556:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941483.737559:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941483.737560:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90277800 x1860112241157248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941499 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941483.737563:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941483.737567:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941483.737571:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941483.737575:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941483.737579:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241157376:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000010:1.0:1773941483.737581:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e300. 00000100:00000001:1.0:1773941483.737586:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941483.737588:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941483.737590:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941483.737593:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:1.0:1773941483.737596:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:2.0:1773941483.737597:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941483.737603:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941483.737603:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941483.737608:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000010:2.0:1773941483.737611:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce4c00. 00000020:00000040:1.0:1773941483.737611:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484600 refcount=4 00000020:00000001:1.0:1773941483.737617:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:1.0:1773941483.737622:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00010000:1.0:1773941483.737625:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d6a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941483.737626:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941483.737633:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941483.737638:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941483.737646:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00000001:1.0:1773941483.737647:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000200:2.0:1773941483.737651:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241157376, portal 25 00010000:00010000:1.0:1773941483.737653:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d6a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941483.737657:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941483.737660:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121082112 : -128249588469504 : ffff8b5b90d63100) 00010000:00000001:1.0:1773941483.737662:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941483.737666:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d6a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941483.737670:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63100 x1860112241157376/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941483.737692:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941483.737694:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941483.737695:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941483.737699:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d6a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941483.737700:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941483.737730:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941483.737732:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941483.737735:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941483.737738:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d6a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941483.737744:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81484600 already unused, calling callback (ffffffffc1482c20) 00000400:00000010:2.0:1773941483.737745:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34198. 10000000:00000001:1.0:1773941483.737747:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941483.737751:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d6a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941483.737753:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241157376, offset 0 00010000:00000001:1.0:1773941483.737758:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941483.737759:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941483.737761:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941483.737762:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484600 refcount=5 00000020:00000001:1.0:1773941483.737765:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:1.0:1773941483.737767:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:1.0:1773941483.737769:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:2.0:1773941483.737771:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941483.737772:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d6a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:1.0:1773941483.737779:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941483.737781:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00010000:1.0:1773941483.737782:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d6a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941483.737789:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941483.737790:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941483.737795:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00000040:1.0:1773941483.737799:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:2.0:1773941483.737802:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 10000000:00000001:1.0:1773941483.737803:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941483.737808:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:2.0:1773941483.737809:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941483.737811:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941483.737815:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941483.737816:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941483.737819:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:2.0:1773941483.737823:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[102, 103] 00010000:00000001:1.0:1773941483.737823:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941483.737826:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941483.737830:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81484600 with handle 0xa33a5be14eae1d6a from hash 00010000:00000001:1.0:1773941483.737834:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:2.0:1773941483.737835:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941483.737836:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941483.737839:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941483.737841:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000400:00000200:2.0:1773941483.737844:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [209:209:256:4294967295] 192.168.204.143@tcp LPNI seq info [104:104:8:4294967295] 00010000:00000001:1.0:1773941483.737844:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941483.737847:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941483.737851:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941483.737853:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941483.737855:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941483.737856:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941483.737861:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941483.737863:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d6a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941483.737873:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941483.737874:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941483.737877:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941483.737879:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941483.737883:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941483.737885:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941483.737886:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941483.737887:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941483.737889:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941483.737893:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d6a lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:2.0:1773941483.737895:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0800. 00010000:00000040:1.0:1773941483.737902:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962200 count: 0 00000800:00000200:2.0:1773941483.737906:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941483.737916:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:2.0:1773941483.737917:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941483.737919:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941483.737923:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000800:00000200:2.0:1773941483.737924:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0800 type 1, nob 616 niov 1 nkiov 1 00000100:00000040:1.0:1773941483.737933:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90277800 x1860112241157248/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941499 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941483.737939:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941483.737941:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941483.737942:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941483.737946:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90262000. 00000400:00000200:2.0:1773941483.737951:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 02000000:00000001:1.0:1773941483.737952:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941483.737955:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941483.737956:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294924101 00000020:00000040:1.0:1773941483.737958:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941483.737963:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941483.737965:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294924099 02000000:00000010:1.0:1773941483.737966:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e3000. 00000100:00000001:2.0:1773941483.737975:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294924101 02000000:00000001:1.0:1773941483.737975:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941483.737977:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941483.737981:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277800. 00000100:00000001:2.0:1773941483.737983:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294924099 00000100:00000001:1.0:1773941483.737984:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941483.737987:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941483.737990:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941483.737991:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:01000000:1.0:1773941483.737993:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:2.0:1773941483.737997:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941483.737998:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000800:00000200:3.0:1773941483.738001:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941483.738003:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000010:1.0:1773941483.738004:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000800:00000010:3.0:1773941483.738005:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0800. 00000100:00000001:2.0:1773941483.738010:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1773941483.738011:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000400:00000200:3.0:1773941483.738013:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941483.738019:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000020:00000010:1.0:1773941483.738020:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000100:00000001:2.0:1773941483.738021:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:3.0:1773941483.738023:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34198 00000100:00000001:2.0:1773941483.738026:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000040:00000010:1.0:1773941483.738026:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000400:00000010:3.0:1773941483.738027:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34198. 00000100:00000001:3.0:1773941483.738034:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1773941483.738035:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:3.0:1773941483.738039:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63100 x1860112241157376/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941483.738045:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000100:00000001:2.0:1773941483.738046:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941483.738051:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941483.738051:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941483.738055:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 00000100:00000040:3.0:1773941483.738057:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63100 x1860112241157376/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941493 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941483.738057:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 80000000:00000010:1.0:1773941483.738062:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000001:3.0:1773941483.738070:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:1.0:1773941483.738070:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 00000100:00000001:3.0:1773941483.738086:0:2347:0:(events.c:68:request_out_callback()) Process leaving 20000000:00000010:1.0:1773941483.738092:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941483.738097:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941483.738099:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941483.738104:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941483.738109:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941483.738111:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941483.738115:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941483.738117:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941483.738126:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941483.738138:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941483.738141:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941483.738143:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941483.738146:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941483.738148:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941483.738150:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941483.738152:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941483.738153:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941483.738154:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941483.738160:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941483.739601:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.739607:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941483.739610:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.739613:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941483.739617:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941483.739622:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941483.739626:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941483.739629:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941483.739665:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941483.739668:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606500 00000400:00000200:3.0:1773941483.739670:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941483.739673:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0xfe5 [1] + 0 00000800:00000001:3.0:1773941483.739682:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.739687:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941483.739688:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941483.739691:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941483.739694:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941483.739697:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63100 x1860112241157376/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941493 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941483.739726:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63100 x1860112241157376/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941493 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941483.739736:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941483.739740:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941483.739743:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941483.739774:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941483.739777:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941483.739781:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941483.739783:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941483.739786:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941483.739790:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941483.739793:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941483.739796:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941483.739802:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63100 x1860112241157376/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941493 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941483.739813:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63100 x1860112241157376/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941493 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941483.739822:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941483.739824:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941483.739826:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941483.739828:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941483.739830:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941483.739832:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941483.739834:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941483.739838:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=62 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941483.739842:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941483.739845:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941483.739847:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941483.739850:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941483.739852:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.739856:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941483.739860:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63100 x1860112241157376/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941493 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941483.739869:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941483.739872:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941483.739873:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941483.739876:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941483.739888:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941483.739891:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941483.739893:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941483.739895:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941483.739897:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941483.739900:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941483.739903:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941483.739907:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941483 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941483.739911:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941483.739918:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941483.739921:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941483.739923:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941483.739925:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.739927:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941483.739930:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941483.739934:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941483.739936:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941483.739940:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941483.739944:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941483.739947:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941483.740193:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.740198:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941483.740206:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63100 x1860112241157376/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941493 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941483.740208:0:12207:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:1.0:1773941483.740220:0:12207:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:2.0:1773941483.740222:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241157376:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00100000:1.0:1773941483.740224:0:12207:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941483.740240:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941483.740243:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941483.740245:0:12207:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941483.740248:0:12207:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000040:2.0:1773941483.740250:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63100 x1860112241157376/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941493 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:1.0:1773941483.740252:0:12207:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941483.740255:0:12207:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941483.740259:0:12207:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941483.740263:0:12207:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941483.740265:0:12207:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941483.740267:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:1.0:1773941483.740269:0:12207:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941483.740272:0:12207:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941483.740273:0:12207:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:2.0:1773941483.740277:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:1.0:1773941483.740280:0:12207:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:2.0:1773941483.740282:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce4c00. 00000100:00000001:1.0:1773941483.740286:0:12207:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:2.0:1773941483.740289:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941483.740292:0:12207:0:(recover.c:147:ptlrpc_resend()) Process entered 00000020:00000001:2.0:1773941483.740293:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941483.740296:0:12207:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941483.740297:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941483.740303:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:1.0:1773941483.740303:0:12207:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 02000000:00000010:2.0:1773941483.740308:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737000. 00000100:00000001:1.0:1773941483.740309:0:12207:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941483.740312:0:12207:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 02000000:00000001:2.0:1773941483.740315:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000040:1.0:1773941483.740315:0:12207:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 02000000:00000001:2.0:1773941483.740319:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941483.740323:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63100. 00000100:00000001:2.0:1773941483.740328:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941483.740330:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941483.740333:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941483.740334:0:12207:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:2.0:1773941483.740338:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000001:1.0:1773941483.740338:0:12207:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941483.740342:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941483.740344:0:12207:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941483.740348:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941483.740353:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:02000000:1.0:1773941483.740354:0:12207:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941483.740368:0:12207:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941483.740372:0:12207:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941483.740374:0:12207:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941483.740377:0:12207:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941483.740379:0:12207:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941483.743735:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962200. 00010000:00000010:1.1:1773941483.743742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81484600. 00000020:00000001:0.0:1773941484.383876:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941484.383883:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941484.383889:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941485.087812:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941485.087820:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941485.087824:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941485.087826:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941485.087828:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941485.087829:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941485.087832:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941485.087833:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941485.087835:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941485.087837:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941485.087839:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941485.087848:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90276a00. 00000020:00000040:3.0:1773941485.087852:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941485.087855:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941485.087858:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941485.087863:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941485.087869:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2600. 00000100:00000001:3.0:1773941485.087876:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941485.087879:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941485.087881:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941485.087888:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962600. 00010000:00000001:3.0:1773941485.087893:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941485.087910:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485200. 00000020:00000001:3.0:1773941485.087913:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941485.087915:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485200 with handle 0xa33a5be14eae1d71 to hash 00000020:00000001:3.0:1773941485.087917:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941485.087918:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00010000:00000001:3.0:1773941485.087921:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00010000:00000001:3.0:1773941485.087924:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941485.087927:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941485.087930:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d71 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941485.087938:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d71 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941485.087947:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d71 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941485.087954:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941485.087957:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941485.087964:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e300. 00000100:00000001:3.0:1773941485.087967:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387400448 : -128249322151168 : ffff8b5ba0b5e300) 00000100:00000001:3.0:1773941485.087970:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941485.087971:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109627904 : -128249599923712 : ffff8b5b90276a00) 00000100:00000001:3.0:1773941485.087977:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941485.087981:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941485.087983:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941485.087985:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941485.087996:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90276a00 x1860112241157504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941485.088006:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941485.088008:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941485.088012:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941485.088014:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941485.088027:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941485.088032:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241157504:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941485.088037:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941485.088042:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941485.088045:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941485.088048:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941485.088066:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941485.088069:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941485.088077:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941485.088081:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab058. 00000400:00000010:3.0:1773941485.088088:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6ac38. 00000100:00000200:3.0:1773941485.088095:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241157504, portal 25 00000100:00000001:3.0:1773941485.088098:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941485.088099:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109627904 : -128249599923712 : ffff8b5b90276a00) 00000100:00000040:3.0:1773941485.088106:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90276a00 x1860112241157504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941501 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941485.088114:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941485.088118:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941485.088121:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a5d8. 00000100:00000200:3.0:1773941485.088124:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241157504, offset 0 00000400:00000200:3.0:1773941485.088131:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941485.088140:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941485.088148:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941485.088153:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941485.088158:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941485.088162:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941485.088165:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941485.088168:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[104, 103] 00000400:00000200:3.0:1773941485.088172:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941485.088176:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [210:210:256:4294967295] 192.168.204.153@tcp LPNI seq info [105:105:8:4294967295] 00000400:00000200:3.0:1773941485.088181:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941485.088191:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941485.088205:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941485.088211:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69500. 00000800:00000200:3.0:1773941485.088218:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941485.088226:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941485.088231:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69500 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941485.088258:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941485.088264:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941485.088267:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294925451 00000100:00000001:3.0:1773941485.088274:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294924101 00000100:00000001:3.0:1773941485.088280:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294925451 00000100:00000001:3.0:1773941485.088286:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294924101 00000100:00000001:3.0:1773941485.088290:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941485.088294:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941485.088297:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941485.088299:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941485.088302:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 11 seconds 00000100:00000001:3.0:1773941485.088305:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941485.088308:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941485.088373:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941485.088378:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69500. 00000400:00000200:2.0:1773941485.088382:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941485.088388:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941485.088393:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a5d8 00000400:00000010:2.0:1773941485.088395:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a5d8. 00000100:00000001:2.0:1773941485.088399:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941485.088403:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90276a00 x1860112241157504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941501 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941485.088411:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941485.088414:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90276a00 x1860112241157504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941501 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941485.088420:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941485.088421:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941485.088981:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941485.088991:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941485.088993:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941485.088995:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941485.088999:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941485.089003:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941485.089007:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941485.089009:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941485.089069:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941485.089074:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606580 00000400:00000200:3.0:1773941485.089077:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941485.089081:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0xff5 [1] + 224 00000800:00000001:3.0:1773941485.089086:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941485.089097:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941485.089099:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941485.089102:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941485.089106:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941485.089109:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90276a00 x1860112241157504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941501 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941485.089120:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90276a00 x1860112241157504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941501 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941485.089146:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941485.089153:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941485.089157:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941485.089221:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941485.089228:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941485.089230:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941485.089233:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab058. 00000400:00000200:1.0:1773941485.089239:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6ac38 00000400:00000010:1.0:1773941485.089241:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6ac38. 00000100:00000001:1.0:1773941485.089244:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941485.089251:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90276a00 x1860112241157504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941501 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941485.089263:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90276a00 x1860112241157504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941501 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941485.089271:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941485.089273:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941485.089275:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941485.089278:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941485.089281:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941485.089283:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941485.089286:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941485.089292:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=64 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941485.089296:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941485.089300:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941485.089304:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941485.089307:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941485.089311:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941485.089314:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941485.089318:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941485.089323:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941485.089327:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941485.089337:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941485.089338:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941485.089341:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941485.089345:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90276a00 x1860112241157504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941501 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941485.089354:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941485.089355:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941485.089357:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941485.089360:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941485.089363:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941485.089365:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941485.089367:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941485.089368:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941485.089370:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941485.089371:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941485.089373:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941485.089376:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941485.089378:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941485.089381:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941485.089384:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 254 00000400:00000200:1.0:1773941485.089390:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941485.089395:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941485.089399:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941485.089403:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941485.089405:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941485.089408:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941485.089411:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941485.089413:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941485.089416:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941485.089418:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941485.089421:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941485.089424:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941485.089426:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941485.089435:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941485.089437:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941485.089440:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941485.089443:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941485.089446:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941485.089449:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941485.089452:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941485.089454:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941485.089457:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941485.089459:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941485.089462:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941485.089467:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63100. 00000020:00000040:1.0:1773941485.089470:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941485.089473:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941485.089475:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941485.089478:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941485.089480:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941485.089482:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941485.089488:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0735400. 00000100:00000001:1.0:1773941485.089492:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941485.089497:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63100 x1860112241157632/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941485.089506:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941485.089509:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941485.089514:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000025727364] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63100 x1860112241157632/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941485.089537:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941485.089541:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941485.089543:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941485.089546:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941485.089548:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941485.089550:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941485.089558:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e300 going to sleep for 1 seconds 00000100:00000001:1.0:1773941485.089560:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941485.089563:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941485.089568:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90276a00 x1860112241157504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941501 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941485.089576:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941485.089579:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941485.089582:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941485.089584:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941485.089588:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90276a00 x1860112241157504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941501 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941485.089598:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241157504:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941485.089603:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941485.089605:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941485.089607:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941485.089610:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941485.089612:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941485.089613:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941485.089617:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941485.089622:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63100 x1860112241157632/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941485.089626:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90276a00 x1860112241157504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941501 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941485.089641:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941485.089646:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e300. 00000100:00000001:1.0:1773941485.089653:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941485.089654:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941485.089656:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941485.089657:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941485.089662:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:1.0:1773941485.089664:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:2.0:1773941485.089665:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941485.089670:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:2.0:1773941485.089671:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941485.089675:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00100000:2.0:1773941485.089678:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241157632:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000020:00000040:1.0:1773941485.089680:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485200 refcount=4 00000100:00000001:2.0:1773941485.089681:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941485.089685:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1773941485.089687:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 02000000:00000001:2.0:1773941485.089688:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941485.089691:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941485.089695:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 02000000:00000010:2.0:1773941485.089696:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce7400. 02000000:00000001:2.0:1773941485.089700:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941485.089738:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941485.089741:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d71 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941485.089742:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941485.089748:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000200:2.0:1773941485.089753:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241157632, portal 25 00000100:00000001:2.0:1773941485.089756:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941485.089757:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121082112 : -128249588469504 : ffff8b5b90d63100) 00000100:00000040:2.0:1773941485.089764:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63100 x1860112241157632/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941495 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941485.089786:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941485.089788:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:2.0:1773941485.089789:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941485.089793:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34220. 00000100:00000200:2.0:1773941485.089797:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241157632, offset 0 00010000:00010000:1.0:1773941485.089798:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d71 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941485.089806:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941485.089814:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941485.089815:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:2.0:1773941485.089819:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00010000:1.0:1773941485.089820:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d71 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941485.089825:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941485.089831:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941485.089837:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941485.089840:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941485.089840:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941485.089844:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941485.089848:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d71 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941485.089853:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[104, 105] 00000400:00000200:2.0:1773941485.089863:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941485.089868:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941485.089869:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [211:211:256:4294967295] 192.168.204.143@tcp LPNI seq info [106:106:8:4294967295] 00010000:00000001:1.0:1773941485.089874:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941485.089877:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941485.089881:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941485.089887:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00010000:1.0:1773941485.089887:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d71 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941485.089896:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941485.089902:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0c00. 00010000:00010000:1.0:1773941485.089904:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485200 already unused, calling callback (ffffffffc1482c20) 00000800:00000200:2.0:1773941485.089909:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00000001:1.0:1773941485.089910:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941485.089916:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d71 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941485.089920:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941485.089925:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0c00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941485.089934:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941485.089936:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941485.089937:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941485.089939:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485200 refcount=5 00000020:00000001:1.0:1773941485.089942:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00000100:00000001:2.0:1773941485.089943:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941485.089946:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00000400:00000200:2.0:1773941485.089948:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941485.089950:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294925453 00000100:00000001:2.0:1773941485.089955:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294925451 00010000:00000001:1.0:1773941485.089960:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000001:2.0:1773941485.089962:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294925453 00010000:00010000:1.0:1773941485.089966:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d71 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941485.089967:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294925451 00000100:00000001:2.0:1773941485.089972:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941485.089977:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941485.089982:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941485.089984:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:2.0:1773941485.089986:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00010000:1.0:1773941485.089989:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d71 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941485.089993:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941485.089997:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941485.090002:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:3.0:1773941485.090003:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 10000000:01000000:1.0:1773941485.090004:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:2.0:1773941485.090005:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941485.090008:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1773941485.090010:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0c00. 10000000:00000040:1.0:1773941485.090012:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000001:2.0:1773941485.090013:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941485.090016:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941485.090017:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00010000:1.0:1773941485.090024:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941485.090028:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:3.0:1773941485.090032:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00010000:00000001:1.0:1773941485.090033:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:3.0:1773941485.090036:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34220 00010000:00000001:1.0:1773941485.090037:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000010:3.0:1773941485.090040:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34220. 00010000:00000001:1.0:1773941485.090042:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941485.090044:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941485.090046:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941485.090067:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485200 with handle 0xa33a5be14eae1d71 from hash 00010000:00000001:1.0:1773941485.090071:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941485.090072:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941485.090074:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941485.090075:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941485.090077:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941485.090079:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941485.090081:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941485.090082:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941485.090084:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941485.090086:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941485.090090:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d71 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941485.090106:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941485.090108:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941485.090110:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941485.090112:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000200:3.0:1773941485.090115:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63100 x1860112241157632/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941495 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941485.090121:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941485.090125:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941485.090128:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941485.090134:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00010000:1.0:1773941485.090134:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d71 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941485.090142:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63100 x1860112241157632/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941495 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000040:1.0:1773941485.090147:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962600 count: 0 00000100:00000001:3.0:1773941485.090157:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941485.090161:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1773941485.090173:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941485.090177:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941485.090181:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941485.090187:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90276a00 x1860112241157504/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941501 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941485.090195:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941485.090198:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941485.090201:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 02000000:00000001:1.0:1773941485.090204:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941485.090206:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941485.090208:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941485.090210:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941485.090213:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2600. 02000000:00000001:1.0:1773941485.090216:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941485.090217:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941485.090220:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90276a00. 00000100:00000001:1.0:1773941485.090223:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941485.090224:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941485.090238:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941485.090241:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941485.090243:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941485.090247:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941485.090251:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941485.090256:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941485.090260:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941485.090265:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941485.090269:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941485.090274:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941485.090284:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941485.090288:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 10000000:00000040:1.0:1773941485.090293:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941485.090296:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941485.090300:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941485.090304:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941485.090307:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941485.090311:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941485.090314:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941485.090318:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941485.090321:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941485.090324:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941485.090327:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941485.090329:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941485.090332:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941485.090335:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941485.090337:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941485.090338:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941485.090340:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941485.090343:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941485.091649:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941485.091655:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941485.091657:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941485.091659:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941485.091662:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941485.091666:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941485.091670:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941485.091672:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941485.091679:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941485.091681:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606600 00000400:00000200:3.0:1773941485.091684:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941485.091687:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1005 [1] + 0 00000800:00000001:3.0:1773941485.091691:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941485.091694:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941485.091696:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941485.091698:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941485.091701:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941485.091730:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63100 x1860112241157632/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941495 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941485.091739:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63100 x1860112241157632/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941495 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941485.091751:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941485.091756:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941485.091758:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941485.091783:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941485.091786:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941485.091790:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941485.091791:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941485.091794:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941485.091798:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941485.091805:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941485.091808:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941485.091812:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63100 x1860112241157632/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941495 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941485.091820:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63100 x1860112241157632/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941495 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941485.091825:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941485.091826:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941485.091828:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941485.091829:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941485.091830:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941485.091832:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941485.091833:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941485.091836:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=64 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941485.091838:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941485.091840:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941485.091841:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941485.091843:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941485.091845:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941485.091846:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941485.091849:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63100 x1860112241157632/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941495 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941485.091854:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941485.091855:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941485.091857:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941485.091858:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941485.091863:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941485.091864:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941485.091865:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941485.091866:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941485.091868:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941485.091869:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941485.091871:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941485.091873:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941485 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941485.091880:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941485.091885:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941485.091887:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941485.091888:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941485.091889:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941485.091891:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941485.091893:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941485.091895:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941485.091897:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941485.091899:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941485.091902:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941485.091904:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941485.092124:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941485.092127:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941485.092131:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63100 x1860112241157632/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941495 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941485.092137:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241157632:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941485.092141:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941485.092143:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941485.092145:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63100 x1860112241157632/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941495 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941485.092148:0:12208:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941485.092151:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941485.092154:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941485.092156:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce7400. 02000000:00000001:2.0:1773941485.092159:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941485.092161:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00080000:1.0:1773941485.092161:0:12208:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000020:00000040:2.0:1773941485.092164:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941485.092168:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941485.092170:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0735400. 00000100:00100000:1.0:1773941485.092170:0:12208:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941485.092175:0:12208:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941485.092176:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:01000000:1.0:1773941485.092180:0:12208:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941485.092181:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941485.092183:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63100. 00000001:00000001:1.0:1773941485.092186:0:12208:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941485.092189:0:12208:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941485.092192:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941485.092194:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941485.092195:0:12208:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941485.092196:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941485.092202:0:12208:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941485.092203:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941485.092206:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941485.092206:0:12208:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941485.092210:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941485.092212:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941485.092212:0:12208:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941485.092219:0:12208:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941485.092222:0:12208:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941485.092227:0:12208:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941485.092229:0:12208:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941485.092239:0:12208:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941485.092241:0:12208:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941485.092244:0:12208:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941485.092246:0:12208:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941485.092248:0:12208:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941485.092250:0:12208:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941485.092268:0:12208:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941485.092270:0:12208:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941485.092272:0:12208:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941485.092276:0:12208:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941485.092280:0:12208:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941485.092282:0:12208:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941485.092285:0:12208:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941485.092288:0:12208:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941485.092289:0:12208:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941485.095746:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962600. 00010000:00000010:1.1:1773941485.095753:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485200. 00000020:00000001:0.0:1773941485.407875:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941485.407881:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941485.407885:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1773941486.431858:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941486.431865:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941486.431870:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941487.071769:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941487.071778:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941487.071782:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941487.071785:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941487.071787:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941487.071788:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941487.071791:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941487.071793:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941487.071795:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941487.071798:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941487.071800:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941487.071812:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277480. 00000020:00000040:3.0:1773941487.071817:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941487.071822:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941487.071825:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941487.071831:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941487.071841:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e3000. 00000100:00000001:3.0:1773941487.071848:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941487.071870:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941487.071873:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941487.071881:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962100. 00010000:00000001:3.0:1773941487.071887:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941487.071890:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81484200. 00000020:00000001:3.0:1773941487.071893:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941487.071896:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81484200 with handle 0xa33a5be14eae1d78 to hash 00000020:00000001:3.0:1773941487.071898:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941487.071899:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860122112 : -128249849429504 : ffff8b5b81484200) 00010000:00000001:3.0:1773941487.071902:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860122112 : -128249849429504 : ffff8b5b81484200) 00010000:00000001:3.0:1773941487.071906:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941487.071909:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941487.071912:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1d78 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941487.071922:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1d78 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941487.071938:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1d78 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941487.071943:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941487.071945:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941487.071951:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941487.071953:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941487.071956:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941487.071957:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109630592 : -128249599921024 : ffff8b5b90277480) 00000100:00000001:3.0:1773941487.071962:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941487.071967:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941487.071969:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941487.071970:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941487.071981:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90277480 x1860112241157760/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941487.071988:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941487.071994:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941487.071998:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941487.072000:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941487.072002:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941487.072005:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241157760:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941487.072009:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941487.072042:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941487.072044:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941487.072047:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941487.072052:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90262000. 02000000:00000001:3.0:1773941487.072053:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941487.072062:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941487.072065:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941487.072073:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a908. 00000100:00000200:3.0:1773941487.072080:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241157760, portal 25 00000100:00000001:3.0:1773941487.072082:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941487.072083:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109630592 : -128249599921024 : ffff8b5b90277480) 00000100:00000040:3.0:1773941487.072087:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90277480 x1860112241157760/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941503 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941487.072093:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941487.072096:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941487.072098:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a2a8. 00000100:00000200:3.0:1773941487.072101:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241157760, offset 0 00000400:00000200:3.0:1773941487.072106:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941487.072115:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941487.072139:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941487.072144:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941487.072148:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941487.072152:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941487.072154:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941487.072157:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[106, 105] 00000400:00000200:3.0:1773941487.072159:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941487.072162:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [212:212:256:4294967295] 192.168.204.153@tcp LPNI seq info [107:107:8:4294967295] 00000400:00000200:3.0:1773941487.072171:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941487.072181:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941487.072186:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941487.072190:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69f00. 00000800:00000200:3.0:1773941487.072197:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941487.072205:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941487.072210:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69f00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941487.072227:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941487.072230:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941487.072233:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294927435 00000100:00000001:3.0:1773941487.072240:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294925453 00000100:00000001:3.0:1773941487.072245:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294927435 00000100:00000001:3.0:1773941487.072248:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294925453 00000100:00000001:3.0:1773941487.072252:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941487.072254:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941487.072257:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941487.072259:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941487.072262:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941487.072266:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941487.072270:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941487.072402:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941487.072410:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69f00. 00000400:00000200:2.0:1773941487.072416:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941487.072423:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941487.072429:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a2a8 00000400:00000010:2.0:1773941487.072431:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a2a8. 00000100:00000001:2.0:1773941487.072436:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941487.072443:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277480 x1860112241157760/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941503 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941487.072453:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941487.072456:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90277480 x1860112241157760/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941503 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941487.072461:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941487.072463:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941487.073561:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941487.073574:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941487.073577:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941487.073579:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941487.073615:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941487.073621:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941487.073626:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941487.073629:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941487.073633:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941487.073636:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606680 00000400:00000200:3.0:1773941487.073640:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941487.073651:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x1015 [1] + 224 00000800:00000001:3.0:1773941487.073657:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941487.073664:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941487.073666:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941487.073669:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941487.073675:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941487.073679:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277480 x1860112241157760/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941503 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941487.073688:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90277480 x1860112241157760/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941503 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941487.073718:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941487.073726:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941487.073729:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941487.073763:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941487.073771:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941487.073772:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941487.073775:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:1.0:1773941487.073779:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a908 00000400:00000010:1.0:1773941487.073781:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a908. 00000100:00000001:1.0:1773941487.073784:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941487.073789:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277480 x1860112241157760/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941503 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941487.073798:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277480 x1860112241157760/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941503 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941487.073803:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941487.073805:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941487.073807:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941487.073810:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941487.073812:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941487.073813:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941487.073815:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941487.073821:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=66 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941487.073824:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941487.073833:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941487.073837:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941487.073840:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941487.073843:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941487.073845:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941487.073849:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941487.073853:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941487.073856:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941487.073857:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941487.073859:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941487.073860:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941487.073863:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90277480 x1860112241157760/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941503 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941487.073868:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941487.073869:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941487.073870:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941487.073873:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941487.073875:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941487.073877:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941487.073878:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941487.073879:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941487.073880:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941487.073881:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941487.073882:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941487.073884:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941487.073886:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941487.073888:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941487.073890:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 255 00000400:00000200:1.0:1773941487.073894:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941487.073897:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941487.073901:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941487.073903:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941487.073909:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941487.073910:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941487.073913:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941487.073915:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941487.073917:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941487.073919:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941487.073920:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941487.073922:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941487.073924:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941487.073926:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941487.073927:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941487.073928:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941487.073931:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941487.073933:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941487.073935:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941487.073937:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941487.073939:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941487.073941:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941487.073943:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941487.073945:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941487.073948:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62d80. 00000020:00000040:1.0:1773941487.073950:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941487.073952:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941487.073954:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941487.073956:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941487.073957:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941487.073959:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941487.073963:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737400. 00000100:00000001:1.0:1773941487.073966:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941487.073970:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d62d80 x1860112241157888/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941487.073977:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941487.073982:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941487.073986:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002abf64a5] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d62d80 x1860112241157888/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941487.074008:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941487.074023:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941487.074025:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941487.074027:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941487.074029:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941487.074030:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941487.074031:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941487.074033:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941487.074035:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941487.074039:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90277480 x1860112241157760/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941503 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941487.074045:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941487.074047:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941487.074049:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941487.074051:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941487.074054:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90277480 x1860112241157760/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941503 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941487.074056:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941487.074062:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1773941487.074063:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241157760:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941487.074066:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941487.074069:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941487.074072:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941487.074076:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000040:2.0:1773941487.074077:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62d80 x1860112241157888/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941487.074078:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941487.074084:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90277480 x1860112241157760/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941503 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941487.074096:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941487.074098:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941487.074099:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941487.074102:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 02000000:00000001:2.0:1773941487.074104:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941487.074106:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:2.0:1773941487.074108:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941487.074111:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941487.074114:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:2.0:1773941487.074117:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241157888:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941487.074119:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:2.0:1773941487.074140:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941487.074140:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941487.074142:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941487.074144:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484200 refcount=4 00000020:00000001:1.0:1773941487.074146:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860122112 : -128249849429504 : ffff8b5b81484200) 00010000:00000001:1.0:1773941487.074151:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860122112 : -128249849429504 : ffff8b5b81484200) 02000000:00000001:2.0:1773941487.074152:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941487.074155:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941487.074156:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1d78 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941487.074159:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941487.074165:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce4000. 00010000:00000001:1.0:1773941487.074167:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:2.0:1773941487.074168:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941487.074172:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1d78 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941487.074178:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941487.074180:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1d78 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941487.074188:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941487.074190:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941487.074193:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:2.0:1773941487.074194:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00010000:00010000:1.0:1773941487.074195:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1d78 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941487.074202:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00000001:1.0:1773941487.074203:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941487.074206:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941487.074209:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241157888, portal 25 00010000:00000001:1.0:1773941487.074210:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000001:2.0:1773941487.074214:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00010000:1.0:1773941487.074214:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1d78 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941487.074216:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121081216 : -128249588470400 : ffff8b5b90d62d80) 00010000:00010000:1.0:1773941487.074225:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81484200 already unused, calling callback (ffffffffc1482c20) 00000100:00000040:2.0:1773941487.074226:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62d80 x1860112241157888/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941497 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941487.074229:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941487.074233:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1d78 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941487.074237:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941487.074242:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941487.074247:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34d48. 00000100:00000200:2.0:1773941487.074250:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241157888, offset 0 00010000:00000001:1.0:1773941487.074250:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941487.074253:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941487.074256:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941487.074259:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000020:00000040:1.0:1773941487.074259:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484200 refcount=5 00000020:00000001:1.0:1773941487.074263:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860122112 : -128249849429504 : ffff8b5b81484200) 00010000:00000001:1.0:1773941487.074266:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860122112 : -128249849429504 : ffff8b5b81484200) 00000400:00000200:2.0:1773941487.074267:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941487.074271:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:2.0:1773941487.074274:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00010000:1.0:1773941487.074276:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1d78 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941487.074280:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941487.074286:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 10000000:00000001:1.0:1773941487.074287:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941487.074307:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1d78 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941487.074308:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941487.074315:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:01000000:1.0:1773941487.074319:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941487.074321:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[106, 107] 10000000:00000040:1.0:1773941487.074323:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941487.074325:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941487.074329:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00010000:1.0:1773941487.074329:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941487.074333:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941487.074337:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [213:213:256:4294967295] 192.168.204.143@tcp LPNI seq info [108:108:8:4294967295] 00010000:00000001:1.0:1773941487.074338:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941487.074340:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941487.074343:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941487.074346:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941487.074346:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941487.074349:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81484200 with handle 0xa33a5be14eae1d78 from hash 00010000:00000001:1.0:1773941487.074352:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941487.074354:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941487.074357:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941487.074360:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941487.074360:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941487.074361:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941487.074364:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941487.074368:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941487.074369:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941487.074369:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941487.074374:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1773941487.074376:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0a00. 00000800:00000200:2.0:1773941487.074383:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00000001:1.0:1773941487.074384:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941487.074390:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1d78 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941487.074400:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941487.074401:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941487.074403:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:2.0:1773941487.074408:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0a00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941487.074408:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941487.074410:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941487.074413:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941487.074416:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941487.074418:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941487.074421:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81484200/0xa33a5be14eae1d78 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941487.074427:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1773941487.074432:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962100 count: 0 00000400:00000200:2.0:1773941487.074435:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941487.074440:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294927437 00000100:00000001:2.0:1773941487.074447:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294927435 00010000:00000001:1.0:1773941487.074453:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941487.074454:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294927437 00010000:00000001:1.0:1773941487.074456:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941487.074461:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941487.074463:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294927435 00000100:00000040:1.0:1773941487.074466:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90277480 x1860112241157760/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941503 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941487.074475:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941487.074475:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941487.074478:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941487.074482:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941487.074483:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90262000. 02000000:00000001:1.0:1773941487.074489:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1773941487.074490:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941487.074492:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941487.074496:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000800:00000200:3.0:1773941487.074497:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941487.074498:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1773941487.074499:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941487.074503:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e3000. 00000800:00000010:3.0:1773941487.074504:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0a00. 02000000:00000001:1.0:1773941487.074509:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:3.0:1773941487.074512:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:1.0:1773941487.074512:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941487.074516:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277480. 00000100:00000001:2.0:1773941487.074519:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:3.0:1773941487.074521:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:1.0:1773941487.074522:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941487.074525:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1773941487.074526:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941487.074528:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34d48 10000000:00000001:1.0:1773941487.074531:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941487.074533:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:01000000:1.0:1773941487.074534:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:2.0:1773941487.074537:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000001:1.0:1773941487.074538:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:2.0:1773941487.074542:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941487.074545:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000010:1.0:1773941487.074549:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941487.074554:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941487.074559:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941487.074563:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000400:00000010:3.0:1773941487.074566:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34d48. 00000020:00000010:1.0:1773941487.074569:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000100:00000001:3.0:1773941487.074570:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:1.0:1773941487.074576:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 00000100:00000200:3.0:1773941487.074577:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62d80 x1860112241157888/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941497 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 80000000:00000010:1.0:1773941487.074583:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000001:3.0:1773941487.074587:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00040000:00000010:1.0:1773941487.074590:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 00000100:00000040:3.0:1773941487.074591:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62d80 x1860112241157888/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941497 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 20000000:00000010:1.0:1773941487.074596:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 00000100:00000001:3.0:1773941487.074600:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941487.074603:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000040:1.0:1773941487.074603:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941487.074608:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941487.074613:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941487.074617:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941487.074620:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941487.074624:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941487.074627:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941487.074631:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941487.074633:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941487.074635:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941487.074637:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941487.074639:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941487.074641:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941487.074643:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941487.074644:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941487.074645:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941487.074646:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941487.074648:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941487.075943:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941487.075950:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941487.075952:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941487.075955:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941487.075994:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941487.076003:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941487.076007:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941487.076009:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941487.076027:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941487.076030:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606700 00000400:00000200:3.0:1773941487.076033:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941487.076035:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1025 [1] + 0 00000800:00000001:3.0:1773941487.076039:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941487.076044:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941487.076046:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941487.076048:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941487.076051:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941487.076054:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62d80 x1860112241157888/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941497 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941487.076062:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d62d80 x1860112241157888/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941497 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941487.076073:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941487.076077:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941487.076079:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941487.076111:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941487.076115:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941487.076119:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941487.076120:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941487.076123:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941487.076144:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941487.076146:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941487.076149:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941487.076155:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62d80 x1860112241157888/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941497 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941487.076165:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62d80 x1860112241157888/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941497 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941487.076172:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941487.076174:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941487.076176:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941487.076185:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941487.076187:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941487.076189:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941487.076191:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941487.076195:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=66 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941487.076198:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941487.076201:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941487.076203:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941487.076206:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941487.076208:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941487.076227:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941487.076233:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62d80 x1860112241157888/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941497 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941487.076240:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941487.076243:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941487.076245:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941487.076248:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941487.076253:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941487.076255:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941487.076256:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941487.076258:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941487.076260:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941487.076263:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941487.076266:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941487.076270:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941487 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941487.076273:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941487.076281:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941487.076284:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941487.076286:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941487.076288:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941487.076290:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941487.076294:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941487.076297:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941487.076300:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941487.076312:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941487.076316:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941487.076319:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941487.076567:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941487.076570:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941487.076574:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62d80 x1860112241157888/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941497 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941487.076581:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241157888:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941487.076585:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941487.076586:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941487.076590:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62d80 x1860112241157888/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941497 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941487.076590:0:12209:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941487.076597:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941487.076599:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:1.0:1773941487.076603:0:12209:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 02000000:00000010:2.0:1773941487.076607:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce4000. 00000100:00100000:1.0:1773941487.076608:0:12209:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:2.0:1773941487.076612:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941487.076614:0:12209:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000001:2.0:1773941487.076615:0:2351:0:(genops.c:1236:class_import_put()) Process entered 10000000:01000000:1.0:1773941487.076618:0:12209:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000040:2.0:1773941487.076619:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941487.076623:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941487.076624:0:12209:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941487.076625:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737400. 02000000:00000001:2.0:1773941487.076629:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:1.0:1773941487.076630:0:12209:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941487.076631:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941487.076632:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62d80. 00000100:00000001:2.0:1773941487.076635:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941487.076636:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941487.076637:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941487.076640:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941487.076642:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941487.076643:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941487.076644:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941487.076661:0:12209:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941487.076663:0:12209:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941487.076664:0:12209:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941487.076665:0:12209:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941487.076667:0:12209:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941487.076669:0:12209:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941487.076673:0:12209:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941487.076676:0:12209:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941487.076679:0:12209:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941487.076682:0:12209:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941487.076685:0:12209:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941487.076688:0:12209:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941487.076689:0:12209:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941487.076691:0:12209:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941487.076740:0:12209:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941487.076742:0:12209:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941487.076745:0:12209:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941487.076749:0:12209:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941487.076751:0:12209:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941487.076753:0:12209:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941487.076754:0:12209:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941487.076756:0:12209:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941487.076757:0:12209:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941487.079753:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962100. 00010000:00000010:1.1:1773941487.079761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81484200. 00000020:00000001:0.0:1773941487.455902:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941487.455913:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941487.455919:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941488.415838:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941488.415847:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941488.415852:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941488.415861:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941488.415863:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941488.415864:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941488.415870:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941488.415872:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941488.415874:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941488.415876:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941488.415878:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941488.415892:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277100. 00000020:00000040:3.0:1773941488.415898:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941488.415903:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941488.415925:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941488.415932:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941488.415941:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2600. 00000100:00000001:3.0:1773941488.415949:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941488.415953:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941488.415956:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941488.415965:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962000. 00010000:00000001:3.0:1773941488.415970:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941488.415973:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485200. 00000020:00000001:3.0:1773941488.415977:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941488.415980:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485200 with handle 0xa33a5be14eae1d7f to hash 00000020:00000001:3.0:1773941488.415981:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941488.415983:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00010000:00000001:3.0:1773941488.415995:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00010000:00000001:3.0:1773941488.416000:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941488.416003:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941488.416007:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d7f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941488.416017:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d7f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941488.416028:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d7f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941488.416032:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941488.416034:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941488.416041:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e9c0. 00000100:00000001:3.0:1773941488.416043:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387402176 : -128249322149440 : ffff8b5ba0b5e9c0) 00000100:00000001:3.0:1773941488.416045:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941488.416046:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109629696 : -128249599921920 : ffff8b5b90277100) 00000100:00000001:3.0:1773941488.416052:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941488.416056:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941488.416058:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941488.416064:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941488.416075:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90277100 x1860112241158016/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941488.416083:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941488.416085:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941488.416088:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941488.416091:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941488.416093:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941488.416095:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241158016:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941488.416099:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941488.416106:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941488.416108:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941488.416111:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941488.416115:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90263000. 02000000:00000001:3.0:1773941488.416116:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941488.416124:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941488.416128:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab4d0. 00000400:00000010:3.0:1773941488.416135:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a000. 00000100:00000200:3.0:1773941488.416142:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241158016, portal 25 00000100:00000001:3.0:1773941488.416143:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941488.416144:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109629696 : -128249599921920 : ffff8b5b90277100) 00000100:00000040:3.0:1773941488.416148:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90277100 x1860112241158016/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941504 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941488.416154:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941488.416157:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941488.416159:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6ab28. 00000100:00000200:3.0:1773941488.416161:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241158016, offset 0 00000400:00000200:3.0:1773941488.416166:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941488.416174:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941488.416182:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941488.416187:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941488.416191:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941488.416194:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941488.416200:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941488.416203:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[108, 107] 00000400:00000200:3.0:1773941488.416206:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941488.416210:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [214:214:256:4294967295] 192.168.204.153@tcp LPNI seq info [109:109:8:4294967295] 00000400:00000200:3.0:1773941488.416214:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941488.416224:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941488.416229:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941488.416233:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69700. 00000800:00000200:3.0:1773941488.416241:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941488.416249:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941488.416253:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69700 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941488.416273:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941488.416276:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941488.416279:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294928779 00000100:00000001:3.0:1773941488.416286:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294927437 00000100:00000001:3.0:1773941488.416291:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294928779 00000100:00000001:3.0:1773941488.416293:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294927437 00000100:00000001:3.0:1773941488.416296:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941488.416298:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941488.416301:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941488.416303:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941488.416306:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941488.416309:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941488.416312:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941488.416411:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941488.416417:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69700. 00000400:00000200:2.0:1773941488.416423:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941488.416447:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941488.416453:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6ab28 00000400:00000010:2.0:1773941488.416456:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6ab28. 00000100:00000001:2.0:1773941488.416461:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941488.416465:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277100 x1860112241158016/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941504 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941488.416475:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941488.416478:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90277100 x1860112241158016/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941504 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941488.416484:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.416486:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941488.417009:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.417023:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941488.417025:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.417027:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941488.417031:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941488.417036:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941488.417041:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941488.417086:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941488.417090:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941488.417094:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606780 00000400:00000200:3.0:1773941488.417098:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941488.417102:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x1035 [1] + 224 00000800:00000001:3.0:1773941488.417109:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.417125:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941488.417127:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941488.417130:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941488.417134:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941488.417138:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277100 x1860112241158016/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941504 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941488.417147:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90277100 x1860112241158016/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941504 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941488.417162:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941488.417168:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.417171:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941488.417211:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941488.417220:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941488.417222:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941488.417225:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab4d0. 00000400:00000200:1.0:1773941488.417231:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a000 00000400:00000010:1.0:1773941488.417233:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a000. 00000100:00000001:1.0:1773941488.417237:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941488.417242:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277100 x1860112241158016/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941504 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941488.417251:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277100 x1860112241158016/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941504 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941488.417257:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941488.417264:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941488.417267:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941488.417269:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941488.417271:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941488.417273:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941488.417275:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941488.417282:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=67 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941488.417286:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941488.417289:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941488.417294:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941488.417297:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941488.417301:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941488.417312:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941488.417317:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941488.417326:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941488.417330:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941488.417333:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941488.417336:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941488.417338:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941488.417343:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90277100 x1860112241158016/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941504 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941488.417353:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941488.417354:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941488.417355:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941488.417358:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941488.417360:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941488.417362:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941488.417363:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941488.417364:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941488.417365:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941488.417366:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941488.417368:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941488.417370:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941488.417371:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941488.417379:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941488.417382:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 257 00000400:00000200:1.0:1773941488.417386:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941488.417390:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941488.417393:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941488.417396:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941488.417398:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941488.417400:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941488.417403:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941488.417404:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941488.417406:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941488.417408:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941488.417409:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941488.417411:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941488.417414:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941488.417415:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941488.417416:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941488.417417:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941488.417419:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941488.417422:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941488.417424:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941488.417426:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941488.417428:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941488.417430:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941488.417432:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941488.417434:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941488.417437:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62680. 00000020:00000040:1.0:1773941488.417439:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941488.417441:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941488.417442:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941488.417444:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941488.417445:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941488.417450:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941488.417454:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737c00. 00000100:00000001:1.0:1773941488.417457:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941488.417462:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d62680 x1860112241158144/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941488.417468:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941488.417470:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941488.417474:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048151604] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d62680 x1860112241158144/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941488.417496:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941488.417498:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941488.417499:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941488.417501:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941488.417503:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941488.417505:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941488.417506:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941488.417508:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941488.417509:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941488.417513:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90277100 x1860112241158016/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941504 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941488.417518:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941488.417520:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941488.417522:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941488.417524:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941488.417527:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90277100 x1860112241158016/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941504 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941488.417533:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241158016:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941488.417536:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941488.417539:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941488.417541:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941488.417546:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941488.417550:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90277100 x1860112241158016/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941504 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941488.417551:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941488.417555:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941488.417559:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941488.417559:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941488.417564:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e9c0. 00000100:00000040:2.0:1773941488.417568:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62680 x1860112241158144/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941488.417569:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941488.417572:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941488.417577:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:2.0:1773941488.417581:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:1.0:1773941488.417581:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:2.0:1773941488.417584:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941488.417584:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941488.417587:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485200 refcount=4 02000000:00000001:2.0:1773941488.417588:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1773941488.417591:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 02000000:00000001:2.0:1773941488.417592:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941488.417595:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 02000000:00000001:2.0:1773941488.417596:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941488.417601:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d7f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1773941488.417602:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241158144:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941488.417609:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941488.417612:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941488.417615:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941488.417616:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:2.0:1773941488.417617:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00010000:1.0:1773941488.417621:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d7f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:2.0:1773941488.417623:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce7400. 02000000:00000001:2.0:1773941488.417628:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941488.417634:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:2.0:1773941488.417635:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941488.417638:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d7f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941488.417639:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941488.417646:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00000001:1.0:1773941488.417647:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000200:2.0:1773941488.417650:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241158144, portal 25 00010000:00000001:1.0:1773941488.417650:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941488.417652:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d7f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941488.417654:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941488.417658:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941488.417660:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.417663:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121079424 : -128249588472192 : ffff8b5b90d62680) 00010000:00000001:1.0:1773941488.417663:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941488.417675:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d7f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941488.417680:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62680 x1860112241158144/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941488.417695:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:1.0:1773941488.417695:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485200 already unused, calling callback (ffffffffc1482c20) 00000100:00000040:2.0:1773941488.417698:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 10000000:00000001:1.0:1773941488.417736:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000010:2.0:1773941488.417740:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34550. 10000000:00010000:1.0:1773941488.417740:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d7f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941488.417745:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241158144, offset 0 00000400:00000200:2.0:1773941488.417753:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941488.417756:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:2.0:1773941488.417760:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941488.417760:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941488.417764:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941488.417765:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000020:00000040:1.0:1773941488.417768:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485200 refcount=5 00000400:00000200:2.0:1773941488.417770:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:1.0:1773941488.417774:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00000400:00000200:2.0:1773941488.417775:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941488.417780:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941488.417781:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860126208 : -128249849425408 : ffff8b5b81485200) 00000400:00000200:2.0:1773941488.417783:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941488.417788:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:2.0:1773941488.417789:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[108, 109] 00000400:00000200:2.0:1773941488.417794:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00010000:1.0:1773941488.417794:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d7f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941488.417799:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [215:215:256:4294967295] 192.168.204.143@tcp LPNI seq info [110:110:8:4294967295] 00000400:00000200:2.0:1773941488.417807:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:1.0:1773941488.417814:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941488.417815:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 10000000:00010000:1.0:1773941488.417820:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d7f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941488.417831:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 10000000:01000000:1.0:1773941488.417831:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941488.417837:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000800:00000010:2.0:1773941488.417838:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0a00. 10000000:00000001:1.0:1773941488.417839:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941488.417844:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000800:00000200:2.0:1773941488.417848:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941488.417848:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941488.417853:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941488.417856:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000800:00000200:2.0:1773941488.417859:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941488.417860:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941488.417863:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941488.417865:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0a00 type 1, nob 616 niov 1 nkiov 1 00000020:00000040:1.0:1773941488.417866:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485200 with handle 0xa33a5be14eae1d7f from hash 00010000:00000001:1.0:1773941488.417870:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:2.0:1773941488.417876:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941488.417877:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941488.417880:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941488.417883:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941488.417883:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941488.417886:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:2.0:1773941488.417887:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294928781 00010000:00000001:1.0:1773941488.417890:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:2.0:1773941488.417894:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294928779 00010000:00000001:1.0:1773941488.417894:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941488.417897:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941488.417899:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.417901:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294928781 10000000:00000001:1.0:1773941488.417902:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941488.417906:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d7f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941488.417907:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294928779 00010000:00000001:1.0:1773941488.417912:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941488.417913:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941488.417916:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941488.417918:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:2.0:1773941488.417919:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941488.417920:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941488.417924:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941488.417924:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941488.417927:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941488.417929:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941488.417930:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485200/0xa33a5be14eae1d7f lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941488.417932:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941488.417940:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000040:1.0:1773941488.417941:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962000 count: 0 00000800:00000200:3.0:1773941488.417942:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941488.417943:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000010:3.0:1773941488.417947:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0a00. 00000100:00000001:2.0:1773941488.417948:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941488.417951:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941488.417954:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941488.417955:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941488.417956:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941488.417956:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941488.417959:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941488.417963:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1773941488.417966:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000040:1.0:1773941488.417970:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90277100 x1860112241158016/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941504 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941488.417972:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34550 00000400:00000010:3.0:1773941488.417976:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34550. 00000100:00000001:1.0:1773941488.417981:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941488.417982:0:2347:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1773941488.417984:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941488.418003:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90263000. 02000000:00000001:1.0:1773941488.418006:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941488.418008:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941488.418009:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941488.418011:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941488.418014:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2600. 02000000:00000001:1.0:1773941488.418021:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941488.418023:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941488.418025:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277100. 00000100:00000001:1.0:1773941488.418028:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941488.418029:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941488.418030:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941488.418033:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941488.418035:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941488.418038:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941488.418043:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941488.418048:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000100:00000200:3.0:1773941488.418050:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62680 x1860112241158144/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000040:00000010:1.0:1773941488.418052:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941488.418058:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000100:00000001:3.0:1773941488.418064:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000010:1.0:1773941488.418066:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 00000100:00000040:3.0:1773941488.418070:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62680 x1860112241158144/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941498 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 80000000:00000010:1.0:1773941488.418072:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941488.418079:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 00000100:00000001:3.0:1773941488.418080:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941488.418083:0:2347:0:(events.c:68:request_out_callback()) Process leaving 20000000:00000010:1.0:1773941488.418085:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 10000000:00000040:1.0:1773941488.418089:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941488.418093:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941488.418097:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941488.418100:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941488.418104:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941488.418108:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941488.418111:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941488.418115:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941488.418117:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941488.418119:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941488.418121:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941488.418123:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941488.418126:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941488.418127:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941488.418129:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941488.418129:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941488.418131:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941488.418137:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:3.0:1773941488.419443:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.419448:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941488.419450:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.419451:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941488.419455:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941488.419458:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941488.419462:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941488.419465:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941488.419497:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941488.419500:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606800 00000400:00000200:3.0:1773941488.419503:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941488.419506:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1045 [1] + 0 00000800:00000001:3.0:1773941488.419509:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.419514:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941488.419516:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941488.419518:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941488.419521:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941488.419524:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62680 x1860112241158144/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941498 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941488.419531:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d62680 x1860112241158144/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941498 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941488.419545:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941488.419549:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.419551:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941488.419578:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941488.419581:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941488.419583:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941488.419585:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941488.419586:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941488.419590:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941488.419591:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941488.419593:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941488.419598:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62680 x1860112241158144/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941498 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941488.419606:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62680 x1860112241158144/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941498 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941488.419611:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941488.419612:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941488.419613:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941488.419615:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941488.419616:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941488.419617:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941488.419618:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941488.419620:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=67 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941488.419623:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941488.419625:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941488.419626:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941488.419628:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941488.419630:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.419632:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941488.419641:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62680 x1860112241158144/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941498 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941488.419646:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941488.419648:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941488.419650:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941488.419652:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941488.419656:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941488.419657:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941488.419658:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941488.419660:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941488.419661:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941488.419663:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941488.419665:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:2.0:1773941488.419668:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941488 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941488.419670:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941488.419676:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:2.0:1773941488.419677:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941488.419679:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941488.419680:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.419681:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941488.419684:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:2.0:1773941488.419686:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941488.419688:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941488.419690:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941488.419693:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:2.0:1773941488.419695:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941488.419917:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.419920:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941488.419925:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62680 x1860112241158144/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941498 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941488.419933:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241158144:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941488.419934:0:12211:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:1.0:1773941488.419943:0:12211:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:2.0:1773941488.419946:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941488.419948:0:12211:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941488.419949:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941488.419951:0:12211:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941488.419954:0:12211:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000040:2.0:1773941488.419955:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62680 x1860112241158144/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941498 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:1.0:1773941488.419957:0:12211:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941488.419961:0:12211:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941488.419963:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941488.419966:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941488.419966:0:12211:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941488.419969:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce7400. 00000001:00000001:1.0:1773941488.419970:0:12211:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941488.419973:0:12211:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941488.419974:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941488.419975:0:12211:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941488.419976:0:2351:0:(genops.c:1236:class_import_put()) Process entered 10000000:00000001:1.0:1773941488.419978:0:12211:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941488.419979:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941488.419980:0:12211:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:2.0:1773941488.419982:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941488.420001:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737c00. 02000000:00000001:2.0:1773941488.420005:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:1.0:1773941488.420006:0:12211:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:2.0:1773941488.420007:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941488.420010:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62680. 00000100:00000001:1.0:1773941488.420011:0:12211:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941488.420013:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941488.420015:0:12211:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941488.420016:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941488.420018:0:12211:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.420019:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941488.420023:0:12211:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941488.420024:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941488.420026:0:12211:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941488.420027:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941488.420028:0:12211:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:2.0:1773941488.420030:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000040:1.0:1773941488.420031:0:12211:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:2.0:1773941488.420033:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941488.420051:0:12211:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941488.420055:0:12211:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941488.420057:0:12211:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941488.420061:0:12211:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941488.420064:0:12211:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941488.420067:0:12211:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941488.420069:0:12211:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941488.420071:0:12211:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941488.420074:0:12211:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941488.423730:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962000. 00010000:00000010:1.1:1773941488.423737:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485200. 00000020:00000001:0.0:1773941488.479883:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941488.479893:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941488.479900:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1773941488.543828:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1773941488.543845:0:37:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1773941488.543859:0:37:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021ca80. 00000020:00000040:2.0:1773941488.543867:0:37:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941488.543874:0:37:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941488.543878:0:37:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941488.543885:0:37:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941488.543895:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b98bb0c00. 00000100:00000001:2.0:1773941488.543903:0:37:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941488.543916:0:37:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS req@ffff8b5b9021ca80 x1860112241158272/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941488.543927:0:37:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941488.543950:0:37:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941488.543957:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000005f3b654] to pc [ptlrpcd_01_01+1] req@ffff8b5b9021ca80 x1860112241158272/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941488.544004:0:37:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941488.544009:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (264) 00000100:00000001:2.0:1773941488.544024:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1773941488.544029:0:2354:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:2.0:1773941488.544032:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941488.544038:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941488.544043:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.544045:0:2354:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941488.544048:0:2354:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941488.544050:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941488.544052:0:2354:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941488.544057:0:2354:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941488.544062:0:2354:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021ca80 x1860112241158272/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941488.544068:0:2354:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941488.544070:0:2354:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941488.544074:0:2354:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941488.544077:0:2354:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941488.544079:0:2354:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941488.544082:0:2354:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:1080e6b5-6a6c-4762-b0ec-c785990fec17:2354:1860112241158272:192.168.204.143@tcp:400:kworker.0 00000100:00000001:2.0:1773941488.544085:0:2354:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941488.544091:0:2354:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941488.544093:0:2354:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941488.544095:0:2354:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941488.544100:0:2354:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b90e7f400. 02000000:00000001:2.0:1773941488.544101:0:2354:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941488.544109:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941488.544113:0:2354:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00000400:00000010:2.0:1773941488.544120:0:2354:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000200:2.0:1773941488.544128:0:2354:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241158272, portal 25 00000100:00000001:2.0:1773941488.544130:0:2354:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941488.544131:0:2354:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109259392 : -128249600292224 : ffff8b5b9021ca80) 00000100:00000040:2.0:1773941488.544135:0:2354:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021ca80 x1860112241158272/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941488.544145:0:2354:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941488.544148:0:2354:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941488.544150:0:2354:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34f68. 00000100:00000200:2.0:1773941488.544153:0:2354:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241158272, offset 0 00000400:00000200:2.0:1773941488.544158:0:2354:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941488.544168:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941488.544178:0:2354:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941488.544185:0:2354:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941488.544189:0:2354:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941488.544194:0:2354:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941488.544197:0:2354:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941488.544201:0:2354:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[110, 109] 00000400:00000200:2.0:1773941488.544205:0:2354:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:2.0:1773941488.544211:0:2354:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [216:216:256:4294967295] 192.168.204.153@tcp LPNI seq info [111:111:8:4294967295] 00000400:00000200:2.0:1773941488.544216:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941488.544228:0:2354:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:2.0:1773941488.544235:0:2354:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:2.0:1773941488.544240:0:2354:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0800. 00000800:00000200:2.0:1773941488.544247:0:2354:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:2.0:1773941488.544255:0:2354:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:2.0:1773941488.544259:0:2354:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0800 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1773941488.544274:0:2354:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941488.544277:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941488.544280:0:2354:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294928907 00000100:00000001:2.0:1773941488.544286:0:2354:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294928781 00000100:00000001:2.0:1773941488.544289:0:2354:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294928907 00000100:00000001:2.0:1773941488.544293:0:2354:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294928781 00000100:00000001:2.0:1773941488.544297:0:2354:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941488.544299:0:2354:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.544306:0:2354:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.544308:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941488.544417:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941488.544424:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0800. 00000400:00000200:3.0:1773941488.544432:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941488.544439:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941488.544449:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34f68 00000400:00000010:3.0:1773941488.544452:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34f68. 00000100:00000001:3.0:1773941488.544455:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941488.544466:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021ca80 x1860112241158272/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941488.544475:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941488.544478:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021ca80 x1860112241158272/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941504 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941488.544484:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941488.544486:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941488.544971:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.544996:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941488.544998:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.545001:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941488.545010:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941488.545015:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941488.545060:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941488.545066:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941488.545071:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941488.545077:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606880 00000400:00000200:3.0:1773941488.545082:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941488.545088:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x1055 [1] + 224 00000800:00000001:3.0:1773941488.545097:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.545107:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941488.545109:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941488.545112:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941488.545116:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941488.545120:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021ca80 x1860112241158272/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941504 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941488.545128:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b9021ca80 x1860112241158272/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941504 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941488.545141:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941488.545148:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.545150:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941488.545173:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941488.545176:0:2354:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941488.545182:0:2354:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941488.545184:0:2354:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941488.545187:0:2354:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:2.0:1773941488.545191:0:2354:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:2.0:1773941488.545193:0:2354:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:2.0:1773941488.545196:0:2354:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941488.545202:0:2354:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021ca80 x1860112241158272/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941504 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941488.545213:0:2354:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021ca80 x1860112241158272/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941504 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941488.545221:0:2354:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941488.545223:0:2354:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941488.545226:0:2354:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941488.545230:0:2354:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941488.545232:0:2354:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941488.545235:0:2354:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941488.545237:0:2354:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941488.545243:0:2354:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=67 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941488.545246:0:2354:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941488.545248:0:2354:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941488.545250:0:2354:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941488.545253:0:2354:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941488.545255:0:2354:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941488.545259:0:2354:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941488.545262:0:2354:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941488.545265:0:2354:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941488.545267:0:2354:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941488.545268:0:2354:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941488.545270:0:2354:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941488.545278:0:2354:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b9021ca80 x1860112241158272/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941504 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941488.545284:0:2354:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941488.545285:0:2354:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941488.545287:0:2354:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941488.545290:0:2354:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941488.545293:0:2354:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941488.545294:0:2354:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.545296:0:2354:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941488.545296:0:2354:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941488.545297:0:2354:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941488.545298:0:2354:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.545299:0:2354:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941488.545301:0:2354:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941488.545303:0:2354:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941488.545305:0:2354:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941488.545307:0:2354:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 258 00000400:00000200:2.0:1773941488.545310:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941488.545314:0:2354:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941488.545317:0:2354:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941488.545319:0:2354:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941488.545321:0:2354:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941488.545323:0:2354:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:2.0:1773941488.545325:0:2354:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941488.545327:0:2354:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:2.0:1773941488.545329:0:2354:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941488.545332:0:2354:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:2.0:1773941488.545333:0:2354:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:2.0:1773941488.545335:0:2354:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:2.0:1773941488.545337:0:2354:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941488.545338:0:2354:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941488.545339:0:2354:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:2.0:1773941488.545341:0:2354:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:2.0:1773941488.545346:0:2354:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:2.0:1773941488.545349:0:2354:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:2.0:1773941488.545351:0:2354:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941488.545353:0:2354:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941488.545356:0:2354:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941488.545358:0:2354:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.545360:0:2354:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941488.545361:0:2354:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941488.545364:0:2354:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021d500. 00000020:00000040:2.0:1773941488.545366:0:2354:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:2.0:1773941488.545369:0:2354:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941488.545370:0:2354:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.545373:0:2354:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941488.545374:0:2354:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941488.545376:0:2354:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941488.545380:0:2354:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0ce7400. 00000100:00000001:2.0:1773941488.545383:0:2354:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941488.545387:0:2354:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b9021d500 x1860112241158400/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941488.545393:0:2354:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941488.545395:0:2354:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941488.545398:0:2354:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1896fa] to pc [ptlrpcd_rcv+-1] req@ffff8b5b9021d500 x1860112241158400/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941488.545414:0:2354:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.545416:0:2354:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941488.545418:0:2354:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:2.0:1773941488.545422:0:2354:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021ca80 x1860112241158272/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941504 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941488.545427:0:2354:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941488.545429:0:2354:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941488.545431:0:2354:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941488.545438:0:2354:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021ca80 x1860112241158272/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941504 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941488.545445:0:2354:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:1080e6b5-6a6c-4762-b0ec-c785990fec17:2354:1860112241158272:192.168.204.143@tcp:400:kworker.0 00000100:00000001:2.0:1773941488.545449:0:2354:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941488.545451:0:2354:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941488.545454:0:2354:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021ca80 x1860112241158272/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941504 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941488.545460:0:2354:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941488.545462:0:2354:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1773941488.545465:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:2.0:1773941488.545466:0:2354:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b90e7f400. 02000000:00000001:2.0:1773941488.545469:0:2354:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1773941488.545471:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1773941488.545472:0:2354:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941488.545476:0:2354:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941488.545476:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1773941488.545480:0:2354:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941488.545483:0:2354:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b98bb0c00. 00000100:00000040:0.0:1773941488.545487:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021d500 x1860112241158400/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941488.545488:0:2354:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941488.545491:0:2354:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941488.545493:0:2354:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021ca80. 00000100:00000001:2.0:1773941488.545498:0:2354:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941488.545500:0:2354:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941488.545503:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941488.545509:0:2354:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941488.545509:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941488.545513:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.545514:0:2354:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941488.545516:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1773941488.545518:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1773941488.545520:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941488.545521:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941488.545524:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941488.545528:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241158400:192.168.204.143@tcp:250:ptlrpcd_01_00.0 00000100:00000001:0.0:1773941488.545532:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941488.545535:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941488.545537:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941488.545539:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941488.545544:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0c00. 02000000:00000001:0.0:1773941488.545547:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941488.545559:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941488.545562:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941488.545567:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941488.545571:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241158400, portal 25 00000100:00000001:0.0:1773941488.545574:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941488.545575:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109262080 : -128249600289536 : ffff8b5b9021d500) 00000100:00000040:0.0:1773941488.545582:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021d500 x1860112241158400/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941488.545591:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941488.545594:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941488.545597:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c990. 00000100:00000200:0.0:1773941488.545602:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241158400, offset 0 00000400:00000200:0.0:1773941488.545607:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941488.545612:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941488.545616:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941488.545620:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941488.545623:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941488.545626:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941488.545629:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941488.545632:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[110, 111] 00000400:00000200:0.0:1773941488.545636:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941488.545640:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [217:217:256:4294967295] 192.168.204.143@tcp LPNI seq info [112:112:8:4294967295] 00000400:00000200:0.0:1773941488.545645:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941488.545653:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941488.545657:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941488.545662:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00000800:00000200:0.0:1773941488.545667:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941488.545674:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941488.545678:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:0.0:1773941488.545691:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941488.545694:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941488.545757:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294928909 00000800:00000200:3.0:1773941488.545761:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1773941488.545763:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294928907 00000100:00000001:0.0:1773941488.545768:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294928909 00000800:00000010:3.0:1773941488.545771:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000100:00000001:0.0:1773941488.545774:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294928907 00000400:00000200:3.0:1773941488.545776:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1773941488.545779:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1773941488.545783:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:0.0:1773941488.545784:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941488.545790:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c990 00000100:00000001:0.0:1773941488.545791:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941488.545794:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c990. 00000100:00000001:0.0:1773941488.545795:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941488.545798:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1773941488.545802:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000200:3.0:1773941488.545803:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021d500 x1860112241158400/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941498 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941488.545806:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1773941488.545812:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941488.545815:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941488.545816:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941488.545819:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941488.545820:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021d500 x1860112241158400/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941498 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941488.545821:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941488.545830:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941488.545832:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941488.547277:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.547283:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941488.547285:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.547287:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941488.547291:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941488.547294:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:3.0:1773941488.547299:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941488.547301:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941488.547303:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941488.547306:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606900 00000400:00000200:3.0:1773941488.547309:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941488.547312:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1065 [1] + 0 00000800:00000001:3.0:1773941488.547316:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.547320:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941488.547322:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941488.547397:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941488.547400:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941488.547404:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021d500 x1860112241158400/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941498 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941488.547412:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b9021d500 x1860112241158400/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941498 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941488.547431:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941488.547435:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941488.547438:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941488.547466:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941488.547470:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941488.547473:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941488.547475:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941488.547477:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941488.547482:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941488.547484:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941488.547487:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941488.547493:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021d500 x1860112241158400/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941498 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941488.547505:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021d500 x1860112241158400/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941498 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941488.547513:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941488.547514:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941488.547517:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941488.547519:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941488.547521:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941488.547523:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941488.547525:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941488.547529:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=67 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941488.547533:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941488.547542:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941488.547544:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941488.547547:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941488.547550:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941488.547553:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941488.547557:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021d500 x1860112241158400/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941498 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941488.547564:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941488.547567:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941488.547569:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941488.547571:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941488.547578:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941488.547581:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941488.547583:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941488.547585:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941488.547587:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941488.547589:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941488.547592:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941488.547596:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941488 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941488.547599:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941488.547606:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941488.547608:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941488.547610:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941488.547612:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941488.547614:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941488.547618:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941488.547621:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941488.547623:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941488.547627:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941488.547631:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941488.547634:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941488.547892:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.547895:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941488.547899:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021d500 x1860112241158400/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941498 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941488.547906:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241158400:192.168.204.143@tcp:250:ptlrpcd_01_00.0 00000100:00000001:2.0:1773941488.547911:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941488.547913:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941488.547916:0:12212:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941488.547921:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021d500 x1860112241158400/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941498 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941488.547929:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941488.547932:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:0.0:1773941488.547932:0:12212:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 02000000:00000010:2.0:1773941488.547934:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0c00. 00000100:00100000:0.0:1773941488.547937:0:12212:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:2.0:1773941488.547938:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941488.547941:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941488.547941:0:12212:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941488.547944:0:12212:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000040:2.0:1773941488.547945:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941488.547949:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:0.0:1773941488.547949:0:12212:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941488.547952:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0ce7400. 00000001:00000001:0.0:1773941488.547954:0:12212:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941488.547956:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941488.547958:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:0.0:1773941488.547959:0:12212:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941488.547960:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021d500. 00000001:00000001:0.0:1773941488.547964:0:12212:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941488.547965:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:0.0:1773941488.547966:0:12212:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941488.547968:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:0.0:1773941488.547969:0:12212:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941488.547970:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1773941488.547972:0:12212:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941488.547974:0:12212:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941488.547975:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941488.547977:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941488.547978:0:12212:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941488.547980:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941488.547982:0:12212:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941488.547994:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941488.548000:0:12212:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941488.548003:0:12212:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941488.548005:0:12212:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941488.548007:0:12212:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941488.548008:0:12212:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941488.548010:0:12212:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941488.548025:0:12212:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941488.548026:0:12212:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941488.548027:0:12212:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941488.548029:0:12212:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941488.548032:0:12212:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941488.548034:0:12212:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941488.548035:0:12212:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941488.548037:0:12212:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941488.548039:0:12212:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941489.503891:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941489.503900:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941489.503904:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941489.887830:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941489.887838:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941489.887843:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941489.887846:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941489.887848:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941489.887849:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941489.887852:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941489.887854:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941489.887857:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941489.887861:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941489.887863:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941489.887874:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275880. 00000020:00000040:3.0:1773941489.887881:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941489.887885:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941489.887889:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941489.887894:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941489.887903:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2600. 00000100:00000001:3.0:1773941489.887910:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941489.887915:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941489.887918:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941489.887926:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962c00. 00010000:00000001:3.0:1773941489.887934:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941489.887937:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81484600. 00000020:00000001:3.0:1773941489.887942:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941489.887945:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81484600 with handle 0xa33a5be14eae1d86 to hash 00000020:00000001:3.0:1773941489.887948:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941489.887950:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:3.0:1773941489.887953:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:3.0:1773941489.887957:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941489.887975:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941489.887980:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d86 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941489.888008:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d86 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941489.888021:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d86 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941489.888029:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941489.888031:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941489.888038:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e9c0. 00000100:00000001:3.0:1773941489.888041:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387402176 : -128249322149440 : ffff8b5ba0b5e9c0) 00000100:00000001:3.0:1773941489.888044:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941489.888046:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109623424 : -128249599928192 : ffff8b5b90275880) 00000100:00000001:3.0:1773941489.888051:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941489.888055:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941489.888058:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941489.888060:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941489.888071:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275880 x1860112241158528/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941489.888081:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941489.888084:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941489.888088:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941489.888090:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941489.888094:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941489.888098:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241158528:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941489.888102:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941489.888107:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941489.888110:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941489.888112:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941489.888117:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90262000. 02000000:00000001:3.0:1773941489.888120:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941489.888127:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941489.888137:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab4d0. 00000400:00000010:3.0:1773941489.888144:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a110. 00000100:00000200:3.0:1773941489.888151:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241158528, portal 25 00000100:00000001:3.0:1773941489.888153:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941489.888155:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109623424 : -128249599928192 : ffff8b5b90275880) 00000100:00000040:3.0:1773941489.888162:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275880 x1860112241158528/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941505 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941489.888171:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941489.888174:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941489.888177:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a330. 00000100:00000200:3.0:1773941489.888180:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241158528, offset 0 00000400:00000200:3.0:1773941489.888186:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941489.888194:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941489.888203:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941489.888208:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941489.888212:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941489.888217:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941489.888220:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941489.888223:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[112, 111] 00000400:00000200:3.0:1773941489.888227:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941489.888231:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [218:218:256:4294967295] 192.168.204.153@tcp LPNI seq info [113:113:8:4294967295] 00000400:00000200:3.0:1773941489.888236:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941489.888247:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941489.888254:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941489.888259:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69c00. 00000800:00000200:3.0:1773941489.888266:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941489.888273:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941489.888278:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69c00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941489.888304:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941489.888309:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941489.888312:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294930251 00000100:00000001:3.0:1773941489.888331:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294928909 00000100:00000001:3.0:1773941489.888337:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294930251 00000100:00000001:3.0:1773941489.888342:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294928909 00000100:00000001:3.0:1773941489.888346:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941489.888350:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941489.888353:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941489.888355:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941489.888358:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941489.888361:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941489.888364:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941489.888426:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941489.888433:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69c00. 00000400:00000200:2.0:1773941489.888439:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941489.888444:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941489.888449:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a330 00000400:00000010:2.0:1773941489.888452:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a330. 00000100:00000001:2.0:1773941489.888458:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941489.888464:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275880 x1860112241158528/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941505 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941489.888474:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941489.888479:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275880 x1860112241158528/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941505 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941489.888487:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941489.888489:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1773941489.889049:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941489.889060:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1773941489.889063:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941489.889066:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941489.889072:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941489.889077:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:3.0:1773941489.889126:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941489.889131:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941489.889136:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941489.889141:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606980 00000400:00000200:3.0:1773941489.889146:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941489.889151:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x106d [1] + 224 00000800:00000001:3.0:1773941489.889158:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941489.889166:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941489.889169:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941489.889173:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941489.889178:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941489.889184:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275880 x1860112241158528/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941505 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941489.889196:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275880 x1860112241158528/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941505 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941489.889229:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1773941489.889237:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:3.0:1773941489.889241:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941489.889274:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941489.889286:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941489.889288:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941489.889291:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab4d0. 00000400:00000200:1.0:1773941489.889299:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a110 00000400:00000010:1.0:1773941489.889317:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a110. 00000100:00000001:1.0:1773941489.889321:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941489.889330:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275880 x1860112241158528/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941505 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941489.889339:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275880 x1860112241158528/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941505 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941489.889346:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941489.889347:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941489.889350:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941489.889352:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941489.889354:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941489.889356:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941489.889359:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941489.889365:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=68 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941489.889368:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941489.889371:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941489.889374:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941489.893010:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941489.893014:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941489.893018:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941489.893022:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941489.893026:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941489.897016:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941489.897019:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941489.897020:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941489.897022:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941489.897028:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275880 x1860112241158528/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941505 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941489.897037:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941489.897038:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941489.897041:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941489.897054:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941489.897057:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941489.897059:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941489.897061:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941489.897062:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941489.897062:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941489.897063:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941489.897064:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941489.897067:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941489.897069:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941489.897071:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941489.897074:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 259 00000400:00000200:1.0:1773941489.897081:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941489.897086:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941489.897089:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941489.897092:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941489.897094:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941489.897096:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941489.897099:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941489.897101:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941489.897103:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941489.897105:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941489.897107:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941489.897108:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941489.897110:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941489.897112:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941489.897113:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941489.897114:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941489.897116:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941489.897119:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941489.897121:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941489.897123:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941489.897126:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941489.897131:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941489.897134:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941489.897136:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941489.897143:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d62a00. 00000020:00000040:1.0:1773941489.897147:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941489.897152:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941489.897155:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941489.897158:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941489.897161:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941489.897165:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941489.897173:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734c00. 00000100:00000001:1.0:1773941489.897181:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941489.897188:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d62a00 x1860112241158656/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941489.897199:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941489.897204:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941489.897209:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000acb4b66b] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d62a00 x1860112241158656/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941489.897227:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941489.897229:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941489.897231:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941489.897233:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941489.897236:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941489.897237:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941489.897239:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941489.897242:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941489.897244:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941489.897251:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275880 x1860112241158528/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941505 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941489.897257:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941489.897259:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941489.897263:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941489.897264:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941489.897267:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941489.897286:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941489.897292:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941489.897292:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275880 x1860112241158528/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941505 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941489.897307:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d62a00 x1860112241158656/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941489.897312:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241158528:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941489.897328:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941489.897334:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941489.897335:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941489.897342:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941489.897343:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:2.0:1773941489.897354:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941489.897354:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941489.897363:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941489.897368:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941489.897368:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275880 x1860112241158528/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941505 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941489.897379:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241158656:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941489.897394:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941489.897396:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:1.0:1773941489.897400:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e9c0. 02000000:00000001:2.0:1773941489.897404:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941489.897407:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941489.897407:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941489.897411:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941489.897412:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941489.897419:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce4c00. 00010000:00000001:1.0:1773941489.897419:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:2.0:1773941489.897426:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941489.897427:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941489.897434:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941489.897437:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484600 refcount=4 00000400:00000200:2.0:1773941489.897442:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:1.0:1773941489.897444:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00000400:00000010:2.0:1773941489.897447:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9000. 00010000:00000001:1.0:1773941489.897449:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00000400:00000010:2.0:1773941489.897454:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00010000:00010000:1.0:1773941489.897456:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d86 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941489.897461:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241158656, portal 25 00000100:00000001:2.0:1773941489.897471:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941489.897475:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121080320 : -128249588471296 : ffff8b5b90d62a00) 00010000:00000001:1.0:1773941489.897475:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941489.897490:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d86 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941489.897492:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d62a00 x1860112241158656/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941499 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941489.897522:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:2.0:1773941489.897523:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941489.897530:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941489.897532:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d86 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941489.897533:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f347f8. 00000100:00000200:2.0:1773941489.897539:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241158656, offset 0 00000400:00000200:2.0:1773941489.897551:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941489.897560:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941489.897566:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941489.897576:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941489.897585:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941489.897591:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941489.897594:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941489.897595:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:1.0:1773941489.897598:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d86 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941489.897602:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941489.897618:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[112, 113] 00010000:00000001:1.0:1773941489.897625:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941489.897629:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941489.897631:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941489.897639:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941489.897640:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [219:219:256:4294967295] 192.168.204.143@tcp LPNI seq info [114:114:8:4294967295] 00010000:00010000:1.0:1773941489.897647:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d86 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941489.897658:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941489.897668:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81484600 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941489.897678:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941489.897684:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 10000000:00010000:1.0:1773941489.897685:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d86 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941489.897736:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941489.897738:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941489.897740:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941489.897741:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81484600 refcount=5 00000020:00000001:1.0:1773941489.897744:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:1.0:1773941489.897746:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860123136 : -128249849428480 : ffff8b5b81484600) 00010000:00000001:1.0:1773941489.897749:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941489.897753:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d86 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:1.0:1773941489.897761:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:2.0:1773941489.897762:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 10000000:00010000:1.0:1773941489.897768:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d86 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:2.0:1773941489.897771:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0c00. 10000000:01000000:1.0:1773941489.897781:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000800:00000200:2.0:1773941489.897785:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00000040:1.0:1773941489.897791:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941489.897795:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941489.897796:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00010000:1.0:1773941489.897805:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000800:00000200:2.0:1773941489.897806:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0c00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941489.897812:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941489.897816:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941489.897818:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941489.897820:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941489.897821:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941489.897824:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941489.897824:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81484600 with handle 0xa33a5be14eae1d86 from hash 00010000:00000001:1.0:1773941489.897833:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941489.897836:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941489.897837:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941489.897839:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941489.897841:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294930261 00010000:00000001:1.0:1773941489.897843:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941489.897847:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941489.897849:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:2.0:1773941489.897853:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294930251 00010000:00000001:1.0:1773941489.897853:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941489.897861:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941489.897864:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294930261 00010000:00000001:1.0:1773941489.897864:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941489.897868:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941489.897870:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294930251 00010000:00010000:1.0:1773941489.897874:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d86 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941489.897878:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941489.897891:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941489.897892:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941489.897897:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941489.897899:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:2.0:1773941489.897900:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941489.897902:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:2.0:1773941489.897905:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941489.897905:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941489.897911:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941489.897914:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941489.897916:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81484600/0xa33a5be14eae1d86 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941489.897917:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941489.897921:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000200:3.0:1773941489.897922:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941489.897931:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000040:1.0:1773941489.897933:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962c00 count: 0 00000800:00000010:3.0:1773941489.897934:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0c00. 00000100:00000001:2.0:1773941489.897935:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941489.897945:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941489.897947:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941489.897948:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941489.897973:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00010000:00000001:1.0:1773941489.897977:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941489.897979:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f347f8 00010000:00000001:1.0:1773941489.897980:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000010:3.0:1773941489.897988:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f347f8. 00000100:00000001:1.0:1773941489.897989:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941489.897996:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1773941489.898002:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275880 x1860112241158528/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941505 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941489.898009:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d62a00 x1860112241158656/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941499 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941489.898014:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941489.898020:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941489.898026:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90262000. 02000000:00000001:1.0:1773941489.898032:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941489.898040:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941489.898043:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000040:1.0:1773941489.898044:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941489.898051:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941489.898054:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2600. 00000100:00000040:3.0:1773941489.898056:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d62a00 x1860112241158656/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941499 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941489.898060:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941489.898062:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941489.898065:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275880. 00000100:00000001:1.0:1773941489.898067:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941489.898068:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941489.898070:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941489.898072:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941489.898075:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941489.898078:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941489.898081:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941489.898086:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941489.898089:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941489.898093:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941489.898097:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941489.898101:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941489.898105:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941489.898110:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941489.898114:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941489.898117:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941489.898122:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941489.898131:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941489.898137:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000100:00000001:3.0:1773941489.898142:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941489.898146:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 00000100:00000001:3.0:1773941489.898147:0:2347:0:(events.c:68:request_out_callback()) Process leaving 80000000:00000010:1.0:1773941489.898156:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941489.898167:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941489.898172:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941489.898175:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941489.898179:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941489.898182:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941489.898185:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941489.898188:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941489.898191:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941489.898193:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941489.898195:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941489.898198:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941489.902945:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941489.902969:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941489.902973:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941489.902975:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941489.902991:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941489.902996:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941489.903001:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941489.903005:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941489.903009:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941489.903012:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606a00 00000400:00000200:2.0:1773941489.903015:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941489.903019:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x107d [1] + 0 00000800:00000001:2.0:1773941489.903025:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941489.903031:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941489.903032:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941489.903035:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941489.903040:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941489.903045:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d62a00 x1860112241158656/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941499 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941489.903054:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d62a00 x1860112241158656/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941499 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941489.903070:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941489.903076:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941489.903078:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941489.903107:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941489.903111:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941489.903117:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941489.903119:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941489.903130:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9000. 00000400:00000200:0.0:1773941489.903134:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:0.0:1773941489.903137:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:0.0:1773941489.903141:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941489.903147:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d62a00 x1860112241158656/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941499 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941489.903159:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d62a00 x1860112241158656/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941499 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941489.903167:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941489.903169:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941489.903172:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941489.903175:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941489.903178:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941489.903181:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941489.903184:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941489.903191:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=68 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941489.903196:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941489.903199:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941489.903201:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941489.903204:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941489.903206:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941489.903209:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941489.903214:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d62a00 x1860112241158656/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941499 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941489.903223:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941489.903226:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941489.903228:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941489.903230:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941489.903235:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941489.903237:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941489.903239:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941489.903241:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941489.903243:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941489.903245:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941489.903252:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941489.903255:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941489 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941489.903258:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941489.903263:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941489.903266:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941489.903268:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941489.903270:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941489.903271:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941489.903274:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941489.903276:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941489.903278:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941489.903280:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941489.903283:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941489.903285:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941489.903477:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941489.903480:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941489.903485:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d62a00 x1860112241158656/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941499 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941489.903493:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d62a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241158656:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941489.903498:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941489.903499:0:12213:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941489.903500:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941489.903504:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d62a00 x1860112241158656/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941499 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941489.903509:0:12213:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:2.0:1773941489.903516:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:0.0:1773941489.903516:0:12213:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941489.903530:0:12213:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941489.903533:0:12213:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941489.903534:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941489.903537:0:12213:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941489.903538:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce4c00. 00000001:00000001:0.0:1773941489.903542:0:12213:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941489.903543:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941489.903546:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941489.903548:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941489.903549:0:12213:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941489.903550:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941489.903553:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734c00. 00000001:00000001:0.0:1773941489.903555:0:12213:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941489.903557:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941489.903559:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941489.903560:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d62a00. 00000001:00000001:0.0:1773941489.903562:0:12213:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941489.903564:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941489.903566:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:0.0:1773941489.903567:0:12213:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941489.903569:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1773941489.903572:0:12213:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941489.903576:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941489.903576:0:12213:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941489.903580:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941489.903583:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1773941489.903583:0:12213:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941489.903586:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941489.903588:0:12213:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941489.903590:0:12213:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941489.903593:0:12213:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941489.903595:0:12213:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941489.903597:0:12213:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941489.903598:0:12213:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941489.903605:0:12213:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941489.903621:0:12213:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941489.903623:0:12213:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941489.903624:0:12213:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941489.903627:0:12213:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00010000:00000010:1.1:1773941489.903729:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962c00. 00010000:00000010:1.1:1773941489.903736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81484600. 00000100:00000001:0.0:1773941489.907464:0:12213:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941489.907468:0:12213:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941489.907469:0:12213:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941489.907471:0:12213:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941489.907472:0:12213:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941490.527870:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941490.527876:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941490.527880:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941491.359840:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941491.359850:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941491.359854:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941491.359857:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941491.359858:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941491.359860:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941491.359863:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941491.359865:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941491.359867:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941491.359870:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941491.359871:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941491.359884:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941491.359889:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941491.359894:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941491.359898:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941491.359903:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941491.359910:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941491.359918:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941491.359923:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941491.359926:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941491.359953:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890eca00. 00010000:00000001:1.0:1773941491.359959:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941491.359962:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941491.359967:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941491.359970:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1d8d to hash 00000020:00000001:1.0:1773941491.359990:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941491.359991:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941491.359994:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941491.359998:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941491.360002:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941491.360005:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d8d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941491.360014:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d8d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941491.360025:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d8d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941491.360030:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941491.360032:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941491.360039:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941491.360041:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941491.360043:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941491.360045:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000001:1.0:1773941491.360050:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941491.360054:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941491.360056:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941491.360058:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941491.360069:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241158784/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941491.360077:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941491.360079:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941491.360082:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941491.360084:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941491.360086:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941491.360089:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241158784:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941491.360098:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941491.360103:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941491.360106:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941491.360108:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941491.360112:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0736c00. 02000000:00000001:1.0:1773941491.360114:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941491.360122:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941491.360126:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941491.360133:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941491.360140:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241158784, portal 25 00000100:00000001:1.0:1773941491.360142:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941491.360143:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:1.0:1773941491.360147:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241158784/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941507 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941491.360153:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941491.360156:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941491.360158:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1990. 00000100:00000200:1.0:1773941491.360161:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241158784, offset 0 00000400:00000200:1.0:1773941491.360167:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941491.360175:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941491.360183:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941491.360187:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941491.360192:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941491.360195:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941491.360197:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941491.360200:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[114, 113] 00000400:00000200:1.0:1773941491.360203:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941491.360206:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [220:220:256:4294967295] 192.168.204.153@tcp LPNI seq info [115:115:8:4294967295] 00000400:00000200:1.0:1773941491.360209:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941491.360217:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941491.360223:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941491.360230:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b900. 00000800:00000200:1.0:1773941491.360237:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941491.360244:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941491.360249:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b900 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941491.360268:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941491.360272:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941491.360276:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294931723 00000100:00000001:1.0:1773941491.360284:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294930261 00000100:00000001:1.0:1773941491.360291:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294931723 00000100:00000001:1.0:1773941491.360295:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294930261 00000100:00000001:1.0:1773941491.360300:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941491.360303:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941491.360306:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941491.360309:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941491.360313:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941491.360317:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941491.360322:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941491.360444:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941491.360454:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b900. 00000400:00000200:3.0:1773941491.360462:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941491.360471:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941491.360478:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1990 00000400:00000010:3.0:1773941491.360481:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1990. 00000100:00000001:3.0:1773941491.360487:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941491.360495:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241158784/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941507 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941491.360509:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941491.360513:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d63800 x1860112241158784/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941507 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941491.360522:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941491.360524:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941491.361070:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941491.361082:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941491.361085:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941491.361088:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941491.361095:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941491.361100:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941491.361106:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941491.361110:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941491.361115:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941491.361118:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606a80 00000400:00000200:2.0:1773941491.361122:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941491.361126:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x108d [1] + 224 00000800:00000001:2.0:1773941491.361133:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941491.361139:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941491.361141:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941491.361144:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941491.361148:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941491.361153:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241158784/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941507 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941491.361177:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d63800 x1860112241158784/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941507 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941491.361197:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941491.361205:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941491.361209:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941491.361230:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941491.361237:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941491.361238:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941491.361241:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941491.361245:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941491.361247:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941491.361250:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941491.361254:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241158784/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941507 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941491.361263:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241158784/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941507 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941491.361269:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941491.361270:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941491.361272:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941491.361275:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941491.361277:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941491.361284:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941491.361287:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941491.361293:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=70 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941491.361296:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941491.361298:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941491.361302:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941491.361304:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941491.361307:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941491.361309:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941491.361313:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941491.361317:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941491.361320:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941491.361323:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941491.361324:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941491.361326:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941491.361329:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d63800 x1860112241158784/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941507 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941491.361334:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941491.361335:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941491.361336:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941491.361338:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941491.361341:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941491.361342:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941491.361343:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941491.361344:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941491.361344:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941491.361345:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941491.361346:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941491.361348:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941491.361350:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941491.361352:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941491.361354:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 260 00000400:00000200:1.0:1773941491.361358:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941491.361365:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941491.361368:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941491.361371:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941491.361373:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941491.361375:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941491.361378:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941491.361379:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941491.361381:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941491.361383:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941491.361384:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941491.361386:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941491.361388:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941491.361389:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941491.361390:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941491.361392:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941491.361395:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941491.361397:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941491.361399:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941491.361401:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941491.361403:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941491.361406:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941491.361407:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941491.361409:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941491.361411:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60000. 00000020:00000040:1.0:1773941491.361413:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941491.361416:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941491.361417:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941491.361419:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941491.361420:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941491.361421:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941491.361425:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0735c00. 00000100:00000001:1.0:1773941491.361427:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941491.361431:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60000 x1860112241158912/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941491.361450:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941491.361452:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941491.361456:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000b65cec4] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60000 x1860112241158912/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941491.361475:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941491.361477:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941491.361478:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941491.361481:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941491.361483:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941491.361484:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941491.361485:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941491.361487:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941491.361488:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941491.361492:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241158784/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941507 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941491.361501:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941491.361505:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941491.361508:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941491.361510:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941491.361513:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1773941491.361517:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241158784/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941507 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941491.361518:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941491.361521:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941491.361530:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60000 x1860112241158912/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941491.361536:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241158784:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941491.361544:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941491.361547:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941491.361549:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941491.361550:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941491.361552:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941491.361555:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941491.361556:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241158912:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941491.361561:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941491.361563:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941491.361567:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:2.0:1773941491.361568:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941491.361569:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941491.361570:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941491.361574:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1773941491.361576:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241158784/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941507 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1773941491.361579:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce4c00. 02000000:00000001:2.0:1773941491.361585:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941491.361586:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941491.361590:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941491.361594:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af95d8. 00000100:00000010:1.0:1773941491.361595:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941491.361599:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000400:00000010:2.0:1773941491.361600:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34bb0. 00000100:00000001:1.0:1773941491.361601:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000200:2.0:1773941491.361607:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241158912, portal 25 00010000:00000001:1.0:1773941491.361607:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:2.0:1773941491.361611:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941491.361611:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941491.361614:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941491.361615:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121069568 : -128249588482048 : ffff8b5b90d60000) 00000020:00000040:1.0:1773941491.361617:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 00000020:00000001:1.0:1773941491.361620:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000040:2.0:1773941491.361624:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60000 x1860112241158912/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941501 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941491.361624:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00010000:1.0:1773941491.361631:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d8d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941491.361641:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941491.361645:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:1.0:1773941491.361646:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d8d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941491.361651:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941491.361656:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d8d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941491.361658:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941491.361663:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34ee0. 00010000:00000001:1.0:1773941491.361664:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000200:2.0:1773941491.361669:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241158912, offset 0 00010000:00000001:1.0:1773941491.361669:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941491.361672:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d8d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941491.361678:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941491.361682:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941491.361685:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941491.361686:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941491.361689:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941491.361692:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d8d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941491.361693:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941491.361700:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941491.361724:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941491.361726:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941491.361728:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941491.361731:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[114, 115] 00010000:00010000:1.0:1773941491.361733:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:2.0:1773941491.361735:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941491.361739:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [221:221:256:4294967295] 192.168.204.143@tcp LPNI seq info [116:116:8:4294967295] 00000400:00000200:2.0:1773941491.361747:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:1.0:1773941491.361747:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941491.361751:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d8d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941491.361762:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941491.361764:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941491.361768:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941491.361771:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:2.0:1773941491.361772:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000020:00000040:1.0:1773941491.361773:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000020:00000001:1.0:1773941491.361778:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000800:00000010:2.0:1773941491.361780:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0200. 00010000:00000001:1.0:1773941491.361781:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941491.361787:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000200:2.0:1773941491.361792:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00010000:1.0:1773941491.361793:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d8d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:1.0:1773941491.361800:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:2.0:1773941491.361803:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 10000000:00010000:1.0:1773941491.361803:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d8d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941491.361810:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0200 type 1, nob 616 niov 1 nkiov 1 10000000:01000000:1.0:1773941491.361812:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941491.361815:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941491.361817:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941491.361819:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941491.361821:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:2.0:1773941491.361824:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941491.361825:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941491.361828:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941491.361830:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941491.361831:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941491.361832:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941491.361835:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294931725 00000020:00000040:1.0:1773941491.361836:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1d8d from hash 00010000:00000001:1.0:1773941491.361840:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:2.0:1773941491.361841:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294931723 00010000:00000001:1.0:1773941491.361842:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941491.361846:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941491.361848:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294931725 00010000:00000001:1.0:1773941491.361849:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941491.361852:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941491.361854:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941491.361857:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941491.361858:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294931723 00000100:00000001:2.0:1773941491.361862:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941491.361863:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941491.361865:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941491.361866:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941491.361869:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941491.361870:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941491.361875:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:1.0:1773941491.361875:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d8d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941491.361880:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941491.361882:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941491.361886:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941491.361888:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:2.0:1773941491.361889:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941491.361890:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941491.361893:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941491.361894:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:1.0:1773941491.361895:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941491.361899:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00010000:1.0:1773941491.361899:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d8d lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941491.361903:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941491.361906:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941491.361909:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1773941491.361909:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890eca00 count: 0 00000800:00000200:3.0:1773941491.361923:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941491.361943:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:3.0:1773941491.361945:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0200. 00010000:00000001:1.0:1773941491.361946:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941491.361955:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1773941491.361960:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:1.0:1773941491.361961:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241158784/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941507 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941491.361978:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941491.361982:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:3.0:1773941491.361984:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 02000000:00000010:1.0:1773941491.361986:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0736c00. 02000000:00000001:1.0:1773941491.361991:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:3.0:1773941491.361993:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34ee0 00000020:00000001:1.0:1773941491.361994:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941491.361997:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000400:00000010:3.0:1773941491.362000:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34ee0. 00000020:00000001:1.0:1773941491.362001:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941491.362005:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 00000100:00000001:3.0:1773941491.362008:0:2347:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1773941491.362010:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941491.362014:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941491.362017:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:1.0:1773941491.362022:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000200:3.0:1773941491.362024:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60000 x1860112241158912/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941501 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941491.362025:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941491.362030:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941491.362042:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941491.362047:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941491.362050:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941491.362054:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941491.362059:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941491.362062:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941491.362066:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941491.362070:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941491.362074:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941491.362078:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941491.362083:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 10000000:00000040:1.0:1773941491.362088:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941491.362090:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941491.362104:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000100:00000001:3.0:1773941491.362109:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000040:00000010:1.0:1773941491.362117:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000100:00000040:3.0:1773941491.362119:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60000 x1860112241158912/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941501 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941491.362123:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941491.362131:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941491.362137:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00000100:00000001:3.0:1773941491.362148:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:1.0:1773941491.362148:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941491.362154:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000100:00000001:3.0:1773941491.362156:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1773941491.362159:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941491.362162:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941491.362167:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941491.362169:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941491.362172:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000800:00000001:2.0:1773941491.363272:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941491.363279:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941491.363281:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941491.363283:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941491.363286:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941491.363290:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941491.363294:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941491.363296:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941491.363299:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941491.363302:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606b00 00000400:00000200:2.0:1773941491.363305:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941491.363308:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1095 [1] + 0 00000800:00000001:2.0:1773941491.363311:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941491.363316:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941491.363317:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941491.363319:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941491.363322:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941491.363326:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60000 x1860112241158912/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941501 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941491.363336:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60000 x1860112241158912/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941501 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941491.363357:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941491.363362:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941491.363365:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941491.363391:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941491.363395:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941491.363401:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941491.363403:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941491.363406:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af95d8. 00000400:00000200:0.0:1773941491.363411:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34bb0 00000400:00000010:0.0:1773941491.363414:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34bb0. 00000100:00000001:0.0:1773941491.363417:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941491.363423:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60000 x1860112241158912/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941501 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941491.363435:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60000 x1860112241158912/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941501 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941491.363444:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941491.363446:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941491.363449:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941491.363453:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941491.363455:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941491.363458:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941491.363461:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941491.363468:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=70 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941491.363473:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941491.363475:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941491.363483:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941491.363485:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941491.363487:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941491.363489:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941491.363494:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60000 x1860112241158912/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941501 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941491.363499:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941491.363502:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941491.363503:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941491.363505:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941491.363511:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941491.363513:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941491.363515:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941491.363517:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941491.363519:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941491.363521:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941491.363522:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941491.363525:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941491 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941491.363528:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941491.363533:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941491.363534:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941491.363537:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941491.363539:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941491.363540:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941491.363543:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941491.363544:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941491.363547:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941491.363549:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941491.363552:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941491.363553:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941491.363759:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941491.363762:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941491.363772:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60000 x1860112241158912/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941501 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941491.363781:0:12214:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:0.0:1773941491.363786:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241158912:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941491.363792:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941491.363794:0:12214:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:0.0:1773941491.363795:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1773941491.363799:0:12214:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:0.0:1773941491.363801:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60000 x1860112241158912/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941501 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941491.363807:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941491.363810:0:12214:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:0.0:1773941491.363811:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 10000000:01000000:2.0:1773941491.363814:0:12214:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000010:0.0:1773941491.363816:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce4c00. 00000001:00000001:2.0:1773941491.363817:0:12214:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:0.0:1773941491.363820:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:2.0:1773941491.363822:0:12214:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:0.0:1773941491.363824:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:2.0:1773941491.363826:0:12214:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941491.363826:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:2.0:1773941491.363831:0:12214:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:0.0:1773941491.363832:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:2.0:1773941491.363833:0:12214:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941491.363836:0:12214:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941491.363837:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0735c00. 10000000:00000001:2.0:1773941491.363838:0:12214:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941491.363841:0:12214:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:0.0:1773941491.363841:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941491.363844:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:2.0:1773941491.363848:0:12214:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000010:0.0:1773941491.363848:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60000. 00000100:00000001:2.0:1773941491.363854:0:12214:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941491.363855:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941491.363857:0:12214:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941491.363858:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941491.363860:0:12214:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941491.363861:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941491.363864:0:12214:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941491.363867:0:12214:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941491.363869:0:12214:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:0.0:1773941491.363869:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000040:2.0:1773941491.363870:0:12214:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:0.0:1773941491.363872:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941491.363875:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941491.363876:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941491.363885:0:12214:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941491.363888:0:12214:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941491.363890:0:12214:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941491.363893:0:12214:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941491.363895:0:12214:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941491.363897:0:12214:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941491.363898:0:12214:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941491.363900:0:12214:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941491.363901:0:12214:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941491.366908:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941491.366912:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941491.366914:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941491.366918:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:1.1:1773941491.369749:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890eca00. 00010000:00000010:1.1:1773941491.369761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:0.0:1773941491.551943:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941491.551951:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941491.551956:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941492.447830:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941492.447839:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941492.447844:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941492.447847:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941492.447849:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941492.447868:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941492.447871:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941492.447873:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941492.447876:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941492.447879:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941492.447881:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941492.447888:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941492.447894:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941492.447899:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941492.447903:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941492.447908:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941492.447949:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941492.447956:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941492.447961:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941492.447964:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941492.447971:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890eca00. 00010000:00000001:1.0:1773941492.447976:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941492.447979:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941492.447982:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941492.447985:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1d94 to hash 00000020:00000001:1.0:1773941492.447986:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941492.447988:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941492.447990:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941492.447993:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941492.447997:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941492.447999:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d94 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941492.448007:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d94 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941492.448018:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d94 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941492.448022:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941492.448028:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941492.448033:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941492.448035:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941492.448037:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941492.448038:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000001:1.0:1773941492.448042:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941492.448046:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941492.448048:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941492.448049:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941492.448058:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241159040/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941492.448065:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941492.448066:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941492.448069:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941492.448071:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941492.448073:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941492.448075:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241159040:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941492.448078:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941492.448083:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941492.448085:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941492.448087:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941492.448090:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0736c00. 02000000:00000001:1.0:1773941492.448091:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941492.448097:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941492.448100:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941492.448105:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941492.448111:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241159040, portal 25 00000100:00000001:1.0:1773941492.448113:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941492.448114:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:1.0:1773941492.448117:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241159040/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941508 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941492.448130:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941492.448133:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941492.448141:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1bb0. 00000100:00000200:1.0:1773941492.448144:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241159040, offset 0 00000400:00000200:1.0:1773941492.448149:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941492.448156:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941492.448162:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941492.448166:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941492.448169:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941492.448173:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941492.448175:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941492.448177:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[116, 115] 00000400:00000200:1.0:1773941492.448180:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941492.448183:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [222:222:256:4294967295] 192.168.204.153@tcp LPNI seq info [117:117:8:4294967295] 00000400:00000200:1.0:1773941492.448186:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941492.448193:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941492.448198:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941492.448202:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b200. 00000800:00000200:1.0:1773941492.448207:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941492.448212:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941492.448216:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b200 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941492.448230:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941492.448233:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941492.448235:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294932811 00000100:00000001:1.0:1773941492.448240:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294931725 00000100:00000001:1.0:1773941492.448244:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294932811 00000100:00000001:1.0:1773941492.448247:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294931725 00000100:00000001:1.0:1773941492.448249:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941492.448251:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941492.448253:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941492.448254:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941492.448256:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941492.448262:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941492.448265:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941492.448439:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941492.448450:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b200. 00000400:00000200:3.0:1773941492.448459:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941492.448469:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941492.448477:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1bb0 00000400:00000010:3.0:1773941492.448481:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1bb0. 00000100:00000001:3.0:1773941492.448487:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941492.448495:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241159040/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941508 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941492.448510:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941492.448586:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d63800 x1860112241159040/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941508 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941492.448605:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941492.448607:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941492.449060:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941492.449077:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941492.449080:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941492.449083:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941492.449098:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941492.449105:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941492.449114:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941492.449118:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941492.449123:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941492.449139:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606b80 00000400:00000200:2.0:1773941492.449144:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941492.449151:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x10a5 [1] + 224 00000800:00000001:2.0:1773941492.449158:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941492.449166:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941492.449168:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941492.449171:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941492.449175:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941492.449180:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241159040/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941508 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941492.449188:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d63800 x1860112241159040/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941508 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941492.449205:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941492.449211:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941492.449214:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941492.449243:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941492.449251:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941492.449253:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941492.449256:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941492.449264:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941492.449266:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941492.449270:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941492.449276:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241159040/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941508 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941492.449285:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241159040/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941508 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941492.449300:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941492.449302:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941492.449306:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941492.449308:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941492.449311:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941492.449314:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941492.449317:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941492.449322:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=71 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941492.449326:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941492.449329:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941492.449332:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941492.449334:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941492.449337:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941492.449340:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941492.449343:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941492.449346:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941492.449349:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941492.449351:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941492.449352:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941492.449364:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941492.449367:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d63800 x1860112241159040/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941492.449373:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941492.449374:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941492.449375:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941492.449377:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941492.449379:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941492.449381:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941492.449382:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941492.449383:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941492.449384:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941492.449385:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941492.449386:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941492.449388:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941492.449389:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941492.449391:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941492.449393:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 261 00000400:00000200:1.0:1773941492.449397:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941492.449400:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941492.449403:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941492.449405:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941492.449406:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941492.449408:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941492.449410:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941492.449411:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941492.449413:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941492.449415:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941492.449417:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941492.449418:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941492.449421:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941492.449422:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941492.449423:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941492.449424:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941492.449429:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941492.449432:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941492.449434:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941492.449436:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941492.449438:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941492.449440:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941492.449441:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941492.449443:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941492.449445:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63480. 00000020:00000040:1.0:1773941492.449448:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941492.449450:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941492.449451:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941492.449453:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941492.449454:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941492.449456:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941492.449461:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0735c00. 00000100:00000001:1.0:1773941492.449464:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941492.449468:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63480 x1860112241159168/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941492.449474:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941492.449476:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941492.449479:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000834a8cee] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63480 x1860112241159168/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941492.449500:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941492.449502:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941492.449503:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941492.449506:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941492.449508:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941492.449509:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941492.449511:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941492.449512:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941492.449514:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941492.449517:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241159040/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941492.449529:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941492.449532:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941492.449535:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941492.449537:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941492.449538:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941492.449542:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1773941492.449545:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241159040/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941508 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941492.449545:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941492.449553:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63480 x1860112241159168/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941492.449568:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1773941492.449569:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241159040:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941492.449576:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941492.449579:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941492.449581:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941492.449582:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941492.449585:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941492.449587:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241159168:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941492.449590:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941492.449591:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941492.449594:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941492.449596:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:0.0:1773941492.449596:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941492.449600:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941492.449602:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000010:0.0:1773941492.449604:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0000. 02000000:00000001:0.0:1773941492.449607:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941492.449609:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241159040/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941508 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941492.449614:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941492.449618:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000100:00000001:1.0:1773941492.449623:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1773941492.449625:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000010:1.0:1773941492.449626:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000200:0.0:1773941492.449630:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241159168, portal 25 00000100:00000001:1.0:1773941492.449631:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941492.449635:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941492.449635:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941492.449639:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:0.0:1773941492.449639:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083008 : -128249588468608 : ffff8b5b90d63480) 00010000:00000001:1.0:1773941492.449643:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941492.449646:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:0.0:1773941492.449646:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63480 x1860112241159168/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941502 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1773941492.449650:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 00000020:00000001:1.0:1773941492.449654:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941492.449659:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000001:0.0:1773941492.449661:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941492.449664:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941492.449665:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d94 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941492.449667:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6cf68. 00000100:00000200:0.0:1773941492.449671:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241159168, offset 0 00010000:00000001:1.0:1773941492.449677:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:0.0:1773941492.449681:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941492.449684:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d94 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941492.449686:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941492.449690:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941492.449695:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941492.449699:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941492.449723:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941492.449726:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d94 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941492.449732:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941492.449733:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941492.449735:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d94 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941492.449738:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941492.449743:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:0.0:1773941492.449744:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941492.449747:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941492.449750:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[116, 117] 00010000:00000001:1.0:1773941492.449752:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941492.449757:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d94 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941492.449758:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941492.449765:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [223:223:256:4294967295] 192.168.204.143@tcp LPNI seq info [118:118:8:4294967295] 00010000:00010000:1.0:1773941492.449770:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941492.449777:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941492.449778:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00010000:1.0:1773941492.449782:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d94 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941492.449791:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941492.449800:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000800:00000200:0.0:1773941492.449800:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941492.449804:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000800:00000010:0.0:1773941492.449806:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00000020:00000001:1.0:1773941492.449808:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941492.449813:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000800:00000200:0.0:1773941492.449813:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000020:00000001:1.0:1773941492.449818:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000800:00000200:0.0:1773941492.449822:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941492.449824:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000800:00000200:0.0:1773941492.449826:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941492.449832:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941492.449836:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d94 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941492.449841:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941492.449843:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941492.449845:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941492.449847:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294932813 00000100:00000001:0.0:1773941492.449851:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294932811 10000000:00010000:1.0:1773941492.449852:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d94 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941492.449856:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294932813 00000100:00000001:0.0:1773941492.449861:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294932811 10000000:01000000:1.0:1773941492.449863:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:0.0:1773941492.449866:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:00000040:1.0:1773941492.449867:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941492.449870:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941492.449871:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941492.449876:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:0.0:1773941492.449876:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941492.449880:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:0.0:1773941492.449881:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941492.449884:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941492.449886:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941492.449888:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:0.0:1773941492.449888:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941492.449890:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941492.449891:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000040:1.0:1773941492.449893:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1d94 from hash 00000100:00000001:0.0:1773941492.449895:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941492.449897:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941492.449899:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:0.0:1773941492.449900:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941492.449903:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941492.449903:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941492.449908:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941492.449929:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941492.449931:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941492.449934:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941492.449937:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941492.449938:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941492.449940:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941492.449943:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941492.449946:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d94 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941492.449952:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941492.449954:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941492.449955:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941492.449957:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941492.449959:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941492.449961:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941492.449962:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:3.0:1773941492.449965:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00010000:1.0:1773941492.449965:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d94 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:3.0:1773941492.449971:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000400:00000200:3.0:1773941492.449976:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000040:1.0:1773941492.449978:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890eca00 count: 0 00000400:00000200:3.0:1773941492.449983:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941492.449987:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6cf68 00000400:00000010:3.0:1773941492.449996:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6cf68. 00000100:00000001:3.0:1773941492.450000:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941492.450000:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941492.450005:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000200:3.0:1773941492.450007:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63480 x1860112241159168/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941502 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941492.450011:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941492.450020:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241159040/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941508 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941492.450029:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941492.450031:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941492.450034:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0736c00. 02000000:00000001:1.0:1773941492.450037:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941492.450039:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941492.450041:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941492.450044:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941492.450046:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941492.450050:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941492.450052:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941492.450053:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:1.0:1773941492.450057:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941492.450058:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941492.450060:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941492.450063:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941492.450066:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941492.450071:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941492.450076:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941492.450082:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941492.450088:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000100:00000001:3.0:1773941492.450089:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941492.450095:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63480 x1860112241159168/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941502 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941492.450096:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941492.450103:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 00000100:00000001:3.0:1773941492.450106:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 80000000:00000010:1.0:1773941492.450110:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000001:3.0:1773941492.450111:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00040000:00000010:1.0:1773941492.450117:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941492.450122:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941492.450126:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941492.450128:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941492.450134:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941492.450139:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941492.450149:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941492.450154:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941492.450157:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941492.450162:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941492.450165:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941492.450168:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941492.450171:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941492.450174:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941492.450177:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941492.450180:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941492.450182:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941492.450184:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941492.450186:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941492.450188:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941492.451553:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941492.451563:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941492.451565:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941492.451567:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941492.451571:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941492.451575:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941492.451579:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941492.451582:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941492.451585:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941492.451588:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606c00 00000400:00000200:2.0:1773941492.451591:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941492.451596:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x10ad [1] + 0 00000800:00000001:2.0:1773941492.451601:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941492.451609:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941492.451611:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941492.451615:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941492.451619:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941492.451633:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63480 x1860112241159168/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941502 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941492.451646:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63480 x1860112241159168/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941502 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941492.451668:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941492.451674:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941492.451676:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941492.451697:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941492.451700:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941492.451717:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941492.451719:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941492.451721:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941492.451725:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941492.451726:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941492.451728:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941492.451733:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63480 x1860112241159168/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941502 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941492.451751:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63480 x1860112241159168/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941502 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941492.451756:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941492.451758:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941492.451759:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941492.451761:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941492.451762:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941492.451763:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941492.451765:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941492.451768:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=71 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941492.451770:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941492.451772:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941492.451773:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941492.451775:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941492.451776:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941492.451779:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941492.451783:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63480 x1860112241159168/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941502 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941492.451788:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941492.451790:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941492.451792:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941492.451794:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941492.451798:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941492.451799:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941492.451801:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941492.451802:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941492.451804:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941492.451805:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941492.451807:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941492.451810:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941492 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941492.451812:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941492.451816:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941492.451821:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941492.451823:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941492.451824:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941492.451826:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941492.451828:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941492.451830:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941492.451832:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941492.451834:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941492.451836:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941492.451838:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941492.452120:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941492.452126:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941492.452134:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63480 x1860112241159168/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941502 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941492.452145:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241159168:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941492.452150:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941492.452153:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941492.452157:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63480 x1860112241159168/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941502 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941492.452164:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941492.452166:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941492.452169:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0000. 02000000:00000001:2.0:1773941492.452173:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941492.452175:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941492.452176:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941492.452178:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941492.452180:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0735c00. 02000000:00000001:2.0:1773941492.452183:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941492.452184:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941492.452184:0:12215:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000010:2.0:1773941492.452188:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63480. 00000100:00000001:2.0:1773941492.452191:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941492.452192:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941492.452193:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941492.452196:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941492.452198:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941492.452202:0:12215:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941492.452205:0:12215:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941492.452208:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941492.452208:0:12215:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941492.452212:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:0.0:1773941492.452212:0:12215:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941492.452218:0:12215:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941492.452220:0:12215:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941492.452223:0:12215:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941492.452225:0:12215:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941492.452225:0:12215:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941492.452226:0:12215:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941492.452228:0:12215:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941492.452229:0:12215:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941492.452232:0:12215:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941492.452234:0:12215:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941492.452236:0:12215:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941492.452237:0:12215:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941492.452238:0:12215:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941492.452240:0:12215:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941492.452240:0:12215:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941492.452241:0:12215:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941492.452253:0:12215:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941492.452254:0:12215:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941492.452256:0:12215:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941492.452258:0:12215:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941492.452266:0:12215:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941492.452268:0:12215:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941492.452271:0:12215:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941492.452275:0:12215:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941492.452277:0:12215:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941492.455740:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890eca00. 00010000:00000010:1.1:1773941492.455749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:0.0:1773941492.575859:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941492.575865:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941492.575869:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1773941493.599878:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941493.599886:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941493.599920:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1773941493.663819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1773941493.663827:0:37:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1773941493.663836:0:37:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021d500. 00000020:00000040:2.0:1773941493.663841:0:37:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941493.663845:0:37:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941493.663848:0:37:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941493.663853:0:37:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941493.663860:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b98bb0e00. 00000100:00000001:2.0:1773941493.663866:0:37:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941493.663876:0:37:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS req@ffff8b5b9021d500 x1860112241159296/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941493.663885:0:37:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941493.663902:0:37:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941493.663909:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1896fa] to pc [ptlrpcd_01_00+0] req@ffff8b5b9021d500 x1860112241159296/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941493.663937:0:37:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941493.663941:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (269) 00000100:00000001:3.0:1773941493.663980:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941493.663984:0:2354:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941493.663986:0:2354:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941493.663993:0:2354:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021d500 x1860112241159296/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941493.664000:0:2354:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941493.664002:0:2354:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941493.664005:0:2354:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941493.664007:0:2354:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941493.664009:0:2354:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941493.664011:0:2354:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:1080e6b5-6a6c-4762-b0ec-c785990fec17:2354:1860112241159296:192.168.204.143@tcp:400:kworker.0 00000100:00000001:3.0:1773941493.664015:0:2354:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941493.664019:0:2354:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941493.664021:0:2354:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941493.664023:0:2354:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941493.664028:0:2354:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b839e3000. 02000000:00000001:3.0:1773941493.664029:0:2354:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941493.664036:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941493.664040:0:2354:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941493.664046:0:2354:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941493.664051:0:2354:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241159296, portal 25 00000100:00000001:3.0:1773941493.664053:0:2354:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941493.664054:0:2354:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109262080 : -128249600289536 : ffff8b5b9021d500) 00000100:00000040:3.0:1773941493.664058:0:2354:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021d500 x1860112241159296/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941493.664084:0:2354:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941493.664086:0:2354:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941493.664088:0:2354:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a3b8. 00000100:00000200:3.0:1773941493.664090:0:2354:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241159296, offset 0 00000400:00000200:3.0:1773941493.664096:0:2354:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941493.664103:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941493.664109:0:2354:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941493.664114:0:2354:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941493.664117:0:2354:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941493.664120:0:2354:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941493.664122:0:2354:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941493.664124:0:2354:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[118, 117] 00000400:00000200:3.0:1773941493.664127:0:2354:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941493.664130:0:2354:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [224:224:256:4294967295] 192.168.204.153@tcp LPNI seq info [119:119:8:4294967295] 00000400:00000200:3.0:1773941493.664133:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941493.664140:0:2354:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941493.664144:0:2354:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941493.664148:0:2354:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69600. 00000800:00000200:3.0:1773941493.664153:0:2354:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941493.664160:0:2354:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941493.664163:0:2354:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69600 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1773941493.664176:0:2354:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941493.664179:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941493.664181:0:2354:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294934027 00000100:00000001:3.0:1773941493.664187:0:2354:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294932813 00000100:00000001:3.0:1773941493.664191:0:2354:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294934027 00000100:00000001:3.0:1773941493.664194:0:2354:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294932813 00000100:00000001:3.0:1773941493.664198:0:2354:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941493.664201:0:2354:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941493.664210:0:2354:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941493.664212:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941493.664218:0:2354:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941493.664220:0:2354:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1773941493.664222:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941493.664223:0:2354:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941493.664224:0:2354:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941493.664226:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941493.664237:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941493.664239:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941493.664278:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941493.664282:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69600. 00000400:00000200:2.0:1773941493.664286:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941493.664291:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941493.664294:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a3b8 00000400:00000010:2.0:1773941493.664296:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a3b8. 00000100:00000001:2.0:1773941493.664300:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941493.664303:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021d500 x1860112241159296/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941493.664317:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941493.664322:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021d500 x1860112241159296/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941509 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941493.664328:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941493.664334:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941493.664971:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.664980:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941493.664982:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.664983:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941493.664987:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941493.664992:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941493.664996:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941493.664999:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941493.665002:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.665005:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606c80 00000400:00000200:2.0:1773941493.665008:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.665011:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x10b5 [1] + 224 00000800:00000001:2.0:1773941493.665017:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.665022:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941493.665024:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941493.665027:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941493.665030:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941493.665033:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021d500 x1860112241159296/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941509 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941493.665040:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b9021d500 x1860112241159296/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941509 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941493.665050:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941493.665055:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.665058:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941493.665082:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941493.665085:0:2354:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941493.665089:0:2354:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941493.665091:0:2354:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941493.665093:0:2354:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941493.665096:0:2354:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941493.665097:0:2354:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941493.665099:0:2354:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941493.665104:0:2354:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021d500 x1860112241159296/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941509 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941493.665112:0:2354:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021d500 x1860112241159296/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941509 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941493.665118:0:2354:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941493.665119:0:2354:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941493.665121:0:2354:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941493.665123:0:2354:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941493.665131:0:2354:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941493.665133:0:2354:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941493.665135:0:2354:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941493.665143:0:2354:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=72 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941493.665145:0:2354:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941493.665148:0:2354:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941493.665149:0:2354:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941493.665152:0:2354:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941493.665154:0:2354:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941493.665157:0:2354:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941493.665166:0:2354:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941493.665169:0:2354:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941493.665171:0:2354:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941493.665172:0:2354:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941493.665173:0:2354:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941493.665176:0:2354:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b9021d500 x1860112241159296/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941509 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941493.665182:0:2354:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941493.665183:0:2354:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941493.665184:0:2354:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941493.665187:0:2354:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941493.665189:0:2354:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941493.665190:0:2354:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941493.665191:0:2354:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941493.665192:0:2354:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941493.665193:0:2354:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941493.665194:0:2354:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941493.665195:0:2354:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941493.665196:0:2354:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941493.665198:0:2354:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941493.665200:0:2354:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941493.665202:0:2354:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 262 00000400:00000200:3.0:1773941493.665205:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941493.665208:0:2354:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941493.665210:0:2354:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941493.665212:0:2354:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941493.665214:0:2354:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941493.665216:0:2354:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941493.665218:0:2354:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941493.665220:0:2354:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941493.665221:0:2354:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941493.665224:0:2354:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941493.665225:0:2354:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941493.665230:0:2354:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941493.665232:0:2354:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941493.665233:0:2354:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941493.665235:0:2354:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941493.665236:0:2354:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941493.665238:0:2354:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941493.665240:0:2354:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941493.665242:0:2354:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941493.665244:0:2354:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941493.665246:0:2354:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941493.665248:0:2354:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941493.665249:0:2354:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941493.665251:0:2354:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941493.665254:0:2354:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275500. 00000020:00000040:3.0:1773941493.665256:0:2354:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941493.665258:0:2354:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941493.665259:0:2354:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941493.665261:0:2354:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941493.665262:0:2354:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941493.665264:0:2354:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941493.665267:0:2354:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90260c00. 00000100:00000001:3.0:1773941493.665270:0:2354:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941493.665273:0:2354:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90275500 x1860112241159424/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941493.665279:0:2354:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941493.665281:0:2354:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941493.665284:0:2354:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007829602] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90275500 x1860112241159424/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941493.665298:0:2354:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941493.665299:0:2354:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941493.665301:0:2354:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:3.0:1773941493.665304:0:2354:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021d500 x1860112241159296/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941509 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941493.665313:0:2354:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941493.665316:0:2354:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941493.665317:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941493.665318:0:2354:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941493.665320:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941493.665323:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941493.665324:0:2354:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021d500 x1860112241159296/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941509 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941493.665330:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275500 x1860112241159424/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941493.665337:0:2354:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:1080e6b5-6a6c-4762-b0ec-c785990fec17:2354:1860112241159296:192.168.204.143@tcp:400:kworker.0 00000100:00000001:3.0:1773941493.665347:0:2354:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941493.665348:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941493.665351:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941493.665352:0:2354:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941493.665355:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1773941493.665357:0:2354:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021d500 x1860112241159296/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941509 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941493.665365:0:2354:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941493.665365:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941493.665369:0:2354:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1773941493.665370:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941493.665372:0:2354:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b839e3000. 02000000:00000001:3.0:1773941493.665377:0:2354:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:2.0:1773941493.665377:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241159424:192.168.204.143@tcp:250:ptlrpcd_01_00.0 00000020:00000001:3.0:1773941493.665381:0:2354:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941493.665384:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1773941493.665385:0:2354:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941493.665389:0:2354:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:2.0:1773941493.665389:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941493.665393:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941493.665394:0:2354:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b98bb0e00. 02000000:00000001:2.0:1773941493.665396:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1773941493.665400:0:2354:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000010:2.0:1773941493.665400:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce6c00. 02000000:00000001:3.0:1773941493.665402:0:2354:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1773941493.665404:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941493.665406:0:2354:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021d500. 00000100:00000001:3.0:1773941493.665410:0:2354:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000400:00000200:2.0:1773941493.665410:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941493.665413:0:2354:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1773941493.665414:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9b58. 00000100:00000001:3.0:1773941493.665415:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1773941493.665421:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34990. 00000100:00000001:3.0:1773941493.665422:0:2354:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941493.665425:0:2354:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941493.665427:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241159424, portal 25 00000100:00000001:3.0:1773941493.665428:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941493.665431:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941493.665432:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941493.665434:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109622528 : -128249599929088 : ffff8b5b90275500) 00000100:00000040:2.0:1773941493.665439:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275500 x1860112241159424/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941493.665444:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941493.665446:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941493.665448:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34880. 00000100:00000200:2.0:1773941493.665449:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241159424, offset 0 00000400:00000200:2.0:1773941493.665453:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941493.665457:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.665461:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941493.665464:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941493.665467:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941493.665470:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941493.665472:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941493.665475:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[118, 119] 00000400:00000200:2.0:1773941493.665479:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941493.665482:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [225:225:256:4294967295] 192.168.204.143@tcp LPNI seq info [120:120:8:4294967295] 00000400:00000200:2.0:1773941493.665486:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.665500:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941493.665505:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941493.665509:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0600. 00000800:00000200:2.0:1773941493.665513:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941493.665518:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941493.665521:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0600 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941493.665531:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941493.665533:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941493.665535:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294934028 00000100:00000001:2.0:1773941493.665538:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294934027 00000100:00000001:2.0:1773941493.665541:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294934028 00000100:00000001:2.0:1773941493.665543:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294934027 00000100:00000001:2.0:1773941493.665545:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941493.665547:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941493.665549:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941493.665551:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941493.665555:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941493.665557:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941493.665558:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941493.665559:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941493.665561:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941493.665562:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941493.665588:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941493.665591:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0600. 00000400:00000200:3.0:1773941493.665593:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941493.665597:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941493.665604:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34880 00000400:00000010:3.0:1773941493.665605:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34880. 00000100:00000001:3.0:1773941493.665607:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941493.665610:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275500 x1860112241159424/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941493.665616:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941493.665618:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275500 x1860112241159424/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941503 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941493.665622:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941493.665623:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941493.667266:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.667272:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941493.667274:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.667276:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941493.667280:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.667283:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941493.667288:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.667290:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.667292:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.667295:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606d00 00000400:00000200:2.0:1773941493.667297:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.667307:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x10c5 [1] + 0 00000800:00000001:2.0:1773941493.667312:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.667317:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941493.667319:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941493.667321:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941493.667324:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941493.667327:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275500 x1860112241159424/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941503 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941493.667335:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90275500 x1860112241159424/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941503 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941493.667350:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941493.667354:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.667356:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941493.667391:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941493.667394:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941493.667397:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941493.667398:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941493.667400:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9b58. 00000400:00000200:0.0:1773941493.667404:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34990 00000400:00000010:0.0:1773941493.667405:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34990. 00000100:00000001:0.0:1773941493.667407:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941493.667412:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275500 x1860112241159424/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941503 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941493.667420:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275500 x1860112241159424/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941503 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941493.667425:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941493.667426:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941493.667428:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941493.667429:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941493.667431:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941493.667432:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941493.667433:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941493.667436:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=72 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941493.667439:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941493.667440:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941493.667442:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941493.667444:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941493.667451:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941493.667453:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941493.667456:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275500 x1860112241159424/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941503 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941493.667461:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941493.667463:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941493.667464:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941493.667466:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941493.667471:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941493.667472:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941493.667474:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941493.667475:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941493.667477:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941493.667478:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941493.667480:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941493.667483:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941493 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941493.667485:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941493.667490:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941493.667491:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941493.667493:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941493.667494:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941493.667495:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941493.667498:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941493.667499:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941493.667501:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941493.667504:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941493.667506:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941493.667508:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941493.667700:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941493.667723:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941493.667727:0:12216:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941493.667728:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275500 x1860112241159424/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941503 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941493.667737:0:12216:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:2.0:1773941493.667739:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241159424:192.168.204.143@tcp:250:ptlrpcd_01_00.0 00000100:00100000:0.0:1773941493.667741:0:12216:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941493.667751:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941493.667752:0:12216:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941493.667755:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:01000000:0.0:1773941493.667760:0:12216:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000040:2.0:1773941493.667761:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275500 x1860112241159424/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941503 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000001:00000001:0.0:1773941493.667766:0:12216:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941493.667771:0:12216:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941493.667776:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941493.667779:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941493.667782:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce6c00. 00000001:00000001:0.0:1773941493.667783:0:12216:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941493.667786:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941493.667788:0:12216:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941493.667789:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941493.667791:0:12216:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:2.0:1773941493.667793:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941493.667795:0:12216:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941493.667798:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000001:0.0:1773941493.667800:0:12216:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941493.667802:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90260c00. 00000100:00000001:0.0:1773941493.667804:0:12216:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:2.0:1773941493.667807:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:0.0:1773941493.667814:0:12216:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:2.0:1773941493.667815:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941493.667819:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275500. 00000100:00000001:0.0:1773941493.667821:0:12216:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941493.667822:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941493.667825:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941493.667826:0:12216:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941493.667827:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941493.667830:0:12216:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941493.667833:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941493.667836:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941493.667836:0:12216:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941493.667840:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941493.667841:0:12216:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941493.667843:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:0.0:1773941493.667844:0:12216:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941493.667847:0:12216:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941493.667861:0:12216:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941493.667862:0:12216:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941493.667863:0:12216:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941493.667866:0:12216:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941493.667869:0:12216:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941493.667871:0:12216:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941493.667872:0:12216:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941493.667874:0:12216:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941493.667875:0:12216:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941493.983817:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941493.983825:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941493.983830:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941493.983832:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941493.983834:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941493.983835:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941493.983839:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941493.983841:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941493.983843:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941493.983846:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941493.983848:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941493.983859:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941493.983864:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941493.983870:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941493.983874:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941493.983879:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941493.983901:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941493.983908:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941493.983913:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941493.983916:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941493.983923:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890eca00. 00010000:00000001:1.0:1773941493.983929:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941493.983949:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941493.983954:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941493.983957:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1d9b to hash 00000020:00000001:1.0:1773941493.983960:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941493.983962:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941493.983966:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941493.983970:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941493.983973:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941493.983978:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d9b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941493.983990:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d9b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941493.984003:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d9b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941493.984010:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941493.984012:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941493.984020:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941493.984022:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941493.984025:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941493.984027:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000001:1.0:1773941493.984032:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941493.984037:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941493.984040:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941493.984041:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941493.984052:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241159552/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941493.984061:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941493.984063:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941493.984068:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941493.984071:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941493.984079:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941493.984084:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241159552:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941493.984088:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941493.984094:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941493.984096:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941493.984098:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941493.984115:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734c00. 02000000:00000001:1.0:1773941493.984117:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941493.984129:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941493.984135:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941493.984142:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941493.984150:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241159552, portal 25 00000100:00000001:1.0:1773941493.984152:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941493.984153:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:1.0:1773941493.984160:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241159552/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941509 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941493.984168:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941493.984172:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941493.984175:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1d48. 00000100:00000200:1.0:1773941493.984178:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241159552, offset 0 00000400:00000200:1.0:1773941493.984184:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941493.984193:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941493.984200:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941493.984205:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941493.984210:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941493.984214:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941493.984217:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941493.984221:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[120, 119] 00000400:00000200:1.0:1773941493.984225:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941493.984229:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [226:226:256:4294967295] 192.168.204.153@tcp LPNI seq info [121:121:8:4294967295] 00000400:00000200:1.0:1773941493.984234:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941493.984244:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941493.984257:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941493.984262:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024bb00. 00000800:00000200:1.0:1773941493.984270:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941493.984277:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941493.984282:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024bb00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941493.984304:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941493.984310:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941493.984313:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294934347 00000100:00000001:1.0:1773941493.984321:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294934028 00000100:00000001:1.0:1773941493.984327:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294934347 00000100:00000001:1.0:1773941493.984332:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294934028 00000100:00000001:1.0:1773941493.984336:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941493.984339:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941493.984342:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941493.984344:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941493.984347:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941493.984351:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941493.984354:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941493.984454:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941493.984460:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024bb00. 00000400:00000200:3.0:1773941493.984466:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941493.984473:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941493.984478:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1d48 00000400:00000010:3.0:1773941493.984481:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1d48. 00000100:00000001:3.0:1773941493.984486:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941493.984491:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241159552/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941509 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941493.984503:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941493.984507:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d63800 x1860112241159552/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941509 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941493.984516:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941493.984518:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941493.985021:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.985032:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941493.985035:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.985038:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941493.985043:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941493.985048:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941493.985056:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941493.985059:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941493.985064:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.985068:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606d80 00000400:00000200:2.0:1773941493.985073:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.985080:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x10d5 [1] + 224 00000800:00000001:2.0:1773941493.985086:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.985094:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941493.985096:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941493.985100:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941493.985104:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941493.985110:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241159552/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941509 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941493.985120:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d63800 x1860112241159552/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941509 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941493.985141:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941493.985154:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.985158:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941493.985172:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941493.985179:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941493.985181:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941493.985183:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941493.985186:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941493.985187:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941493.985189:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941493.985193:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241159552/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941509 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941493.985201:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241159552/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941509 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941493.985207:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941493.985208:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941493.985215:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941493.985218:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941493.985220:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941493.985221:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941493.985223:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941493.985228:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=72 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941493.985231:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941493.985233:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941493.985236:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941493.985238:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941493.985241:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941493.985243:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941493.985246:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941493.985249:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941493.985252:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941493.985253:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941493.985254:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941493.985256:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941493.985259:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d63800 x1860112241159552/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941509 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941493.985264:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941493.985265:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941493.985266:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941493.985268:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941493.985270:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941493.985271:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941493.985273:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941493.985273:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941493.985274:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941493.985275:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941493.985276:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941493.985278:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941493.985279:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941493.985281:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941493.985286:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 264 00000400:00000200:1.0:1773941493.985290:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941493.985293:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941493.985296:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941493.985299:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941493.985301:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941493.985302:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941493.985305:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941493.985306:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941493.985308:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941493.985310:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941493.985311:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941493.985313:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941493.985314:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941493.985316:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941493.985317:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941493.985318:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941493.985320:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941493.985322:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941493.985324:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941493.985326:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941493.985328:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941493.985330:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941493.985332:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941493.985333:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941493.985336:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60380. 00000020:00000040:1.0:1773941493.985338:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941493.985341:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941493.985342:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941493.985343:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941493.985345:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941493.985346:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941493.985352:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737c00. 00000100:00000001:1.0:1773941493.985354:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941493.985359:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60380 x1860112241159680/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941493.985364:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941493.985366:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941493.985370:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077e2ee70] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60380 x1860112241159680/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941493.985384:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941493.985386:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941493.985388:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941493.985390:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941493.985391:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941493.985393:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941493.985394:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941493.985395:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941493.985397:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941493.985400:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241159552/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941509 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941493.985405:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941493.985407:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941493.985409:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941493.985411:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941493.985414:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241159552/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941509 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941493.985421:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241159552:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941493.985423:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941493.985424:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941493.985426:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941493.985428:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941493.985429:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941493.985434:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941493.985438:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941493.985439:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241159552/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941509 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941493.985448:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60380 x1860112241159680/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941493.985448:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941493.985453:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941493.985457:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941493.985460:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941493.985465:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:2.0:1773941493.985468:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:1.0:1773941493.985468:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:2.0:1773941493.985473:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941493.985473:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941493.985474:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 02000000:00000001:2.0:1773941493.985479:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1773941493.985479:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 02000000:00000001:2.0:1773941493.985484:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941493.985485:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00010000:1.0:1773941493.985489:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d9b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941493.985491:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941493.985500:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241159680:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941493.985501:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941493.985509:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d9b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941493.985510:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941493.985514:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941493.985517:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941493.985518:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000001:2.0:1773941493.985522:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941493.985528:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce6c00. 00010000:00010000:1.0:1773941493.985531:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d9b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941493.985532:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941493.985541:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941493.985543:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941493.985546:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:2.0:1773941493.985547:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9f20. 00010000:00010000:1.0:1773941493.985549:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d9b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941493.985555:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f344c8. 00010000:00000001:1.0:1773941493.985555:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941493.985558:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941493.985561:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241159680, portal 25 00010000:00000001:1.0:1773941493.985562:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941493.985565:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d9b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941493.985567:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941493.985569:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121070464 : -128249588481152 : ffff8b5b90d60380) 00000100:00000040:2.0:1773941493.985578:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60380 x1860112241159680/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941493.985579:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941493.985584:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941493.985587:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d9b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941493.985596:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941493.985598:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941493.985601:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000040:2.0:1773941493.985602:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000020:00000001:1.0:1773941493.985603:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941493.985607:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000400:00000010:2.0:1773941493.985609:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34770. 00000020:00000001:1.0:1773941493.985610:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941493.985615:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000200:2.0:1773941493.985616:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241159680, offset 0 00010000:00000001:1.0:1773941493.985618:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941493.985622:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d9b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941493.985627:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 10000000:00000001:1.0:1773941493.985632:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941493.985635:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d9b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941493.985636:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:01000000:1.0:1773941493.985643:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941493.985646:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 10000000:00000040:1.0:1773941493.985647:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941493.985650:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941493.985654:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:1.0:1773941493.985654:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941493.985659:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941493.985662:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941493.985663:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941493.985665:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941493.985666:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941493.985667:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941493.985669:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1d9b from hash 00010000:00000001:1.0:1773941493.985673:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941493.985674:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941493.985676:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941493.985677:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941493.985681:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941493.985686:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000400:00000200:2.0:1773941493.985689:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[120, 121] 00010000:00000001:1.0:1773941493.985690:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941493.985692:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941493.985696:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941493.985699:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941493.985701:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941493.985702:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941493.985724:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941493.985726:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d9b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941493.985731:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941493.985733:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941493.985735:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941493.985737:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941493.985738:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941493.985739:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941493.985743:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941493.985747:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1d9b lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941493.985748:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [227:227:256:4294967295] 192.168.204.143@tcp LPNI seq info [122:122:8:4294967295] 00000400:00000200:2.0:1773941493.985762:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000040:1.0:1773941493.985766:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890eca00 count: 0 00000400:00000200:2.0:1773941493.985775:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941493.985782:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941493.985783:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941493.985787:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000010:2.0:1773941493.985789:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0500. 00000100:00000001:1.0:1773941493.985791:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000800:00000200:2.0:1773941493.985802:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000100:00000040:1.0:1773941493.985802:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241159552/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941509 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941493.985812:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941493.985816:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000200:2.0:1773941493.985817:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 02000000:00000010:1.0:1773941493.985819:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734c00. 00000800:00000200:2.0:1773941493.985823:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0500 type 1, nob 616 niov 1 nkiov 1 02000000:00000001:1.0:1773941493.985823:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941493.985826:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941493.985829:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941493.985831:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941493.985833:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941493.985836:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941493.985837:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941493.985842:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941493.985843:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:1.0:1773941493.985847:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000400:00000200:2.0:1773941493.985848:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941493.985849:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941493.985854:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941493.985855:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294934349 10000000:01000000:1.0:1773941493.985857:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941493.985861:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:2.0:1773941493.985863:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294934347 10000000:00000010:1.0:1773941493.985864:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941493.985870:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000100:00000001:2.0:1773941493.985872:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294934349 00000020:00000010:1.0:1773941493.985876:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000100:00000001:2.0:1773941493.985878:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294934347 00000040:00000010:1.0:1773941493.985897:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000100:00000001:2.0:1773941493.985902:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1773941493.985903:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000100:00000001:2.0:1773941493.985908:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941493.985909:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 00000100:00000001:2.0:1773941493.985913:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 80000000:00000010:1.0:1773941493.985915:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000001:2.0:1773941493.985918:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00040000:00000010:1.0:1773941493.985921:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 00000100:00000001:2.0:1773941493.985925:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 20000000:00000010:1.0:1773941493.985927:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 00000100:00000001:2.0:1773941493.985929:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 10000000:00000040:1.0:1773941493.985932:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000100:00000001:2.0:1773941493.985934:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:3.0:1773941493.985935:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 10000000:00000001:1.0:1773941493.985936:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941493.985938:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000800:00000010:3.0:1773941493.985942:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0500. 00000100:00000001:2.0:1773941493.985944:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941493.985944:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000100:00000001:2.0:1773941493.985950:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941493.985951:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000040:00000010:1.0:1773941493.985951:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941493.985957:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000400:00000200:3.0:1773941493.985963:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000020:00000010:1.0:1773941493.985963:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941493.985968:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00000400:00000200:3.0:1773941493.985971:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34770 00040000:00000010:1.0:1773941493.985973:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 00000400:00000010:3.0:1773941493.985975:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34770. 20000000:00000010:1.0:1773941493.985977:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941493.985980:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941493.985982:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941493.985984:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941493.985987:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941493.985988:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941493.985989:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941493.985990:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941493.985992:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941493.985993:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:3.0:1773941493.986017:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941493.986022:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60380 x1860112241159680/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941503 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941493.986047:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941493.986052:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60380 x1860112241159680/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941503 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941493.986057:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941493.986059:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941493.987341:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.987351:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941493.987355:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.987359:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941493.987373:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.987379:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941493.987385:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.987389:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.987392:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.987396:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606e00 00000400:00000200:2.0:1773941493.987401:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941493.987406:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x10dd [1] + 0 00000800:00000001:2.0:1773941493.987411:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.987418:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941493.987421:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941493.987424:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941493.987429:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941493.987435:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60380 x1860112241159680/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941503 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941493.987447:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60380 x1860112241159680/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941503 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941493.987466:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941493.987473:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941493.987476:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941493.987500:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941493.987503:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941493.987507:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941493.987508:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941493.987511:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9f20. 00000400:00000200:0.0:1773941493.987514:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f344c8 00000400:00000010:0.0:1773941493.987516:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f344c8. 00000100:00000001:0.0:1773941493.987518:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941493.987522:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60380 x1860112241159680/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941503 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941493.987529:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60380 x1860112241159680/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941503 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941493.987539:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941493.987541:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941493.987542:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941493.987544:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941493.987545:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941493.987547:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941493.987548:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941493.987552:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=72 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941493.987554:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941493.987556:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941493.987557:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941493.987559:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941493.987560:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941493.987562:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941493.987565:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60380 x1860112241159680/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941503 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941493.987572:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941493.987575:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941493.987576:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941493.987579:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941493.987584:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941493.987586:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941493.987588:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941493.987590:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941493.987592:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941493.987594:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941493.987596:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941493.987600:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941493 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941493.987604:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941493.987610:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941493.987612:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941493.987615:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941493.987617:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941493.987625:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941493.987629:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941493.987631:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941493.987634:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941493.987637:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941493.987639:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941493.987641:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941493.987878:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941493.987898:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941493.987906:0:12217:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941493.987907:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60380 x1860112241159680/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941503 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941493.987923:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241159680:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00080000:0.0:1773941493.987924:0:12217:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941493.987930:0:12217:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941493.987933:0:12217:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941493.987935:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:0.0:1773941493.987935:0:12217:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941493.987942:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000001:00000001:0.0:1773941493.987942:0:12217:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941493.987946:0:12217:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941493.987950:0:12217:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941493.987951:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60380 x1860112241159680/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941503 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:0.0:1773941493.987953:0:12217:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941493.987956:0:12217:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941493.987958:0:12217:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941493.987961:0:12217:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941493.987965:0:12217:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941493.987971:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00080000:0.0:1773941493.987971:0:12217:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941493.987974:0:12217:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941493.987976:0:12217:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941493.987977:0:12217:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941493.987979:0:12217:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941493.987983:0:12217:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941493.987984:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 10000000:01000000:0.0:1773941493.987984:0:12217:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941493.987988:0:12217:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 02000000:00000010:2.0:1773941493.987990:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce6c00. 02000000:00000001:2.0:1773941493.987995:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941493.987998:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941493.988000:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941493.988002:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941493.988005:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737c00. 00000100:00080000:0.0:1773941493.988007:0:12217:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 02000000:00000001:2.0:1773941493.988008:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000001:0.0:1773941493.988010:0:12217:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941493.988012:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941493.988013:0:12217:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000010:2.0:1773941493.988017:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60380. 00000100:02000000:0.0:1773941493.988020:0:12217:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941493.988024:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941493.988026:0:12217:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941493.988027:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1773941493.988030:0:12217:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941493.988033:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941493.988039:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:0.0:1773941493.988039:0:12217:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941493.988044:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941493.988044:0:12217:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941493.988047:0:12217:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941493.988050:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941493.988054:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941493.991739:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890eca00. 00010000:00000010:1.1:1773941493.991749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:3.0:1773941494.623848:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941494.623854:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941494.623857:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941495.071847:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941495.071901:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941495.071906:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941495.071909:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941495.071910:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941495.071912:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941495.071915:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941495.071916:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941495.071919:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941495.071922:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941495.071924:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941495.071932:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941495.071937:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941495.071941:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941495.071945:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941495.071950:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941495.071958:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941495.071965:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941495.071970:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941495.071973:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941495.071981:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890eca00. 00010000:00000001:1.0:1773941495.071988:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941495.071992:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941495.071996:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941495.071999:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1da2 to hash 00000020:00000001:1.0:1773941495.072002:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941495.072004:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941495.072007:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941495.072011:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941495.072014:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941495.072020:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941495.072032:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941495.072057:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941495.072064:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941495.072067:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941495.072074:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941495.072077:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941495.072080:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941495.072082:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000001:1.0:1773941495.072086:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941495.072091:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941495.072093:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941495.072095:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941495.072105:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241159808/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941495.072113:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941495.072115:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941495.072119:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941495.072122:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941495.072125:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941495.072128:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241159808:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941495.072133:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941495.072138:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941495.072140:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941495.072142:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941495.072147:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734c00. 02000000:00000001:1.0:1773941495.072149:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941495.072157:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941495.072162:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941495.072169:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941495.072175:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241159808, portal 25 00000100:00000001:1.0:1773941495.072178:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941495.072185:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:1.0:1773941495.072191:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241159808/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941511 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941495.072200:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941495.072203:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941495.072206:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1550. 00000100:00000200:1.0:1773941495.072210:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241159808, offset 0 00000400:00000200:1.0:1773941495.072216:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941495.072224:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941495.072231:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941495.072236:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941495.072240:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941495.072245:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941495.072248:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941495.072252:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[122, 121] 00000400:00000200:1.0:1773941495.072255:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941495.072260:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [228:228:256:4294967295] 192.168.204.153@tcp LPNI seq info [123:123:8:4294967295] 00000400:00000200:1.0:1773941495.072265:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941495.072275:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941495.072282:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941495.072288:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b600. 00000800:00000200:1.0:1773941495.072295:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941495.072302:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941495.072306:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941495.072324:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941495.072329:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941495.072331:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294935435 00000100:00000001:1.0:1773941495.072340:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294934349 00000100:00000001:1.0:1773941495.072346:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294935435 00000100:00000001:1.0:1773941495.072350:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294934349 00000100:00000001:1.0:1773941495.072361:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941495.072364:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941495.072367:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941495.072369:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941495.072373:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941495.072376:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941495.072379:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941495.072422:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941495.072427:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b600. 00000400:00000200:3.0:1773941495.072432:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941495.072437:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941495.072441:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1550 00000400:00000010:3.0:1773941495.072443:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1550. 00000100:00000001:3.0:1773941495.072446:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941495.072450:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241159808/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941511 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941495.072458:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941495.072461:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d63800 x1860112241159808/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941511 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941495.072466:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941495.072467:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941495.073154:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941495.073165:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941495.073168:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941495.073169:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941495.073174:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941495.073179:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941495.073184:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941495.073186:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941495.073190:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941495.073194:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606e80 00000400:00000200:2.0:1773941495.073197:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941495.073201:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x10ed [1] + 224 00000800:00000001:2.0:1773941495.073205:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941495.073211:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941495.073213:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941495.073216:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941495.073220:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941495.073224:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241159808/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941511 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941495.073243:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d63800 x1860112241159808/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941511 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941495.073256:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941495.073264:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941495.073266:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941495.073300:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941495.073308:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941495.073309:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941495.073311:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941495.073316:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941495.073317:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941495.073320:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941495.073326:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241159808/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941511 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941495.073336:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241159808/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941511 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941495.073343:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941495.073345:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941495.073348:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941495.073350:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941495.073353:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941495.073355:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941495.073357:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941495.073362:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=74 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941495.073366:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941495.073369:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941495.073373:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941495.073376:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941495.073379:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941495.073382:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941495.073386:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941495.073390:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941495.073400:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941495.073403:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941495.073404:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941495.073406:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941495.073411:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d63800 x1860112241159808/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941511 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941495.073418:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941495.073420:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941495.073422:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941495.073425:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941495.073427:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941495.073429:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941495.073431:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941495.073433:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941495.073434:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941495.073435:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941495.073437:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941495.073439:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941495.073441:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941495.073444:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941495.073447:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 264 00000400:00000200:1.0:1773941495.073452:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941495.073456:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941495.073460:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941495.073463:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941495.073465:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941495.073468:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941495.073470:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941495.073472:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941495.073474:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941495.073477:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941495.073479:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941495.073481:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941495.073490:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941495.073492:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941495.073494:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941495.073495:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941495.073499:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941495.073502:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941495.073505:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941495.073508:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941495.073510:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941495.073513:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941495.073515:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941495.073516:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941495.073520:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60a80. 00000020:00000040:1.0:1773941495.073523:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941495.073525:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941495.073527:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941495.073529:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941495.073531:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941495.073533:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941495.073537:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737400. 00000100:00000001:1.0:1773941495.073541:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941495.073546:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60a80 x1860112241159936/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941495.073555:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941495.073557:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941495.073562:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000079bd9b68] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60a80 x1860112241159936/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941495.073583:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941495.073586:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941495.073588:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941495.073590:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941495.073592:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941495.073599:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941495.073601:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941495.073604:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941495.073606:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941495.073612:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241159808/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941511 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941495.073619:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941495.073620:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941495.073626:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941495.073626:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941495.073631:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941495.073632:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941495.073636:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941495.073642:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60a80 x1860112241159936/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941495.073646:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241159808/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941511 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941495.073666:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941495.073669:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941495.073670:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241159808:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941495.073674:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941495.073678:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941495.073679:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941495.073684:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941495.073685:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941495.073689:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941495.073693:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1773941495.073694:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241159936:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000040:1.0:1773941495.073700:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241159808/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941511 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941495.073725:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941495.073730:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941495.073732:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941495.073737:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941495.073744:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce6c00. 00000100:00000001:1.0:1773941495.073745:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941495.073750:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 02000000:00000001:2.0:1773941495.073751:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941495.073757:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941495.073760:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941495.073761:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941495.073766:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000010:2.0:1773941495.073769:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9580. 00010000:00000001:1.0:1773941495.073770:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941495.073775:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:2.0:1773941495.073777:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34a18. 00000020:00000040:1.0:1773941495.073778:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 00000020:00000001:1.0:1773941495.073784:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000200:2.0:1773941495.073785:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241159936, portal 25 00010000:00000001:1.0:1773941495.073787:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000001:2.0:1773941495.073790:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00010000:1.0:1773941495.073793:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da2 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941495.073794:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121072256 : -128249588479360 : ffff8b5b90d60a80) 00000100:00000040:2.0:1773941495.073804:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60a80 x1860112241159936/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941505 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941495.073823:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941495.073825:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:1.0:1773941495.073829:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da2 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941495.073841:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941495.073843:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000010:2.0:1773941495.073847:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f344c8. 00010000:00010000:1.0:1773941495.073849:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da2 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941495.073855:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241159936, offset 0 00010000:00000001:1.0:1773941495.073876:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941495.073881:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941495.073881:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941495.073886:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da2 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941495.073891:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941495.073899:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941495.073900:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:1.0:1773941495.073904:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941495.073907:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941495.073910:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941495.073913:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00010000:1.0:1773941495.073915:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da2 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941495.073919:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941495.073927:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:1.0:1773941495.073930:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:2.0:1773941495.073935:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[122, 123] 10000000:00000001:1.0:1773941495.073937:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941495.073941:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da2 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941495.073944:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941495.073953:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [229:229:256:4294967295] 192.168.204.143@tcp LPNI seq info [124:124:8:4294967295] 00010000:00000001:1.0:1773941495.073957:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941495.073961:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941495.073962:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:1.0:1773941495.073966:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941495.073969:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000020:00000001:1.0:1773941495.073975:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000400:00000200:2.0:1773941495.073978:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941495.073979:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000800:00000200:2.0:1773941495.073985:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941495.073993:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0300. 00010000:00000001:1.0:1773941495.073993:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941495.073998:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da2 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941495.074005:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00000001:1.0:1773941495.074013:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:2.0:1773941495.074018:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 10000000:00010000:1.0:1773941495.074019:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da2 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941495.074025:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0300 type 1, nob 616 niov 1 nkiov 1 10000000:01000000:1.0:1773941495.074031:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941495.074035:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941495.074037:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941495.074041:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:2.0:1773941495.074043:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941495.074043:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941495.074047:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941495.074051:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:2.0:1773941495.074052:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941495.074056:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:2.0:1773941495.074057:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294935437 00010000:00000001:1.0:1773941495.074061:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941495.074065:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294935435 00000020:00000040:1.0:1773941495.074066:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1da2 from hash 00000100:00000001:2.0:1773941495.074074:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294935437 00010000:00000001:1.0:1773941495.074074:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941495.074078:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941495.074081:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294935435 00010000:00000001:1.0:1773941495.074082:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941495.074084:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941495.074089:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:2.0:1773941495.074090:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941495.074091:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941495.074098:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941495.074099:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941495.074101:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941495.074104:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941495.074105:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941495.074110:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941495.074111:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941495.074114:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da2 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941495.074122:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941495.074125:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000010:3.0:1773941495.074128:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0300. 00000100:00000001:2.0:1773941495.074129:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941495.074129:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941495.074133:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:2.0:1773941495.074135:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:1.0:1773941495.074136:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000400:00000200:3.0:1773941495.074138:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941495.074141:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941495.074144:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941495.074148:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941495.074149:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941495.074150:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941495.074152:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941495.074157:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941495.074158:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941495.074159:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f344c8 00010000:00010000:1.0:1773941495.074163:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da2 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941495.074172:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890eca00 count: 0 00010000:00000001:1.0:1773941495.074186:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941495.074188:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941495.074191:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941495.074197:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241159808/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941511 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941495.074205:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941495.074208:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941495.074211:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734c00. 02000000:00000001:1.0:1773941495.074215:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000010:3.0:1773941495.074216:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f344c8. 00000020:00000001:1.0:1773941495.074218:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941495.074222:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941495.074224:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1773941495.074226:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941495.074231:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 00000100:00000200:3.0:1773941495.074233:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60a80 x1860112241159936/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941505 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941495.074236:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941495.074240:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941495.074244:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:3.0:1773941495.074251:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941495.074252:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941495.074254:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941495.074256:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941495.074258:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941495.074263:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000040:3.0:1773941495.074267:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60a80 x1860112241159936/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941505 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000010:1.0:1773941495.074270:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941495.074277:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000100:00000001:3.0:1773941495.074280:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941495.074284:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000100:00000001:3.0:1773941495.074285:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000040:00000010:1.0:1773941495.074291:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941495.074298:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941495.074303:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941495.074308:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941495.074321:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941495.074326:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941495.074330:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941495.074333:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941495.074338:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941495.074341:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941495.074345:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941495.074349:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941495.074352:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941495.074357:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941495.074360:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941495.074363:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941495.074365:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941495.074368:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941495.074371:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941495.074373:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941495.074375:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941495.074377:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941495.074378:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941495.074380:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941495.075483:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941495.075490:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941495.075492:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941495.075493:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941495.075497:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941495.075500:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941495.075504:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941495.075506:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941495.075509:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941495.075511:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321606f00 00000400:00000200:2.0:1773941495.075514:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941495.075517:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x10f5 [1] + 0 00000800:00000001:2.0:1773941495.075520:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941495.075524:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941495.075526:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941495.075528:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941495.075531:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941495.075535:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60a80 x1860112241159936/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941505 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941495.075543:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60a80 x1860112241159936/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941505 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941495.075563:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941495.075569:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941495.075572:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941495.075600:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941495.075603:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941495.075606:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941495.075607:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941495.075609:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9580. 00000400:00000200:0.0:1773941495.075613:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34a18 00000400:00000010:0.0:1773941495.075614:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34a18. 00000100:00000001:0.0:1773941495.075616:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941495.075621:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60a80 x1860112241159936/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941505 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941495.075629:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60a80 x1860112241159936/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941505 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941495.075634:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941495.075635:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941495.075636:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941495.075638:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941495.075639:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941495.075640:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941495.075641:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941495.075644:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=74 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941495.075646:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941495.075648:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941495.075649:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941495.075651:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941495.075653:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941495.075654:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941495.075657:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60a80 x1860112241159936/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941505 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941495.075663:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941495.075664:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941495.075665:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941495.075667:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941495.075676:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941495.075678:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941495.075679:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941495.075680:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941495.075682:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941495.075683:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941495.075685:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941495.075687:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941495 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941495.075689:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941495.075694:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941495.075695:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941495.075697:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941495.075698:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941495.075699:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941495.075717:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941495.075719:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941495.075721:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941495.075724:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941495.075727:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941495.075729:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941495.075954:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941495.075957:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941495.075962:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60a80 x1860112241159936/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941505 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941495.075974:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241159936:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941495.075975:0:12219:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941495.075981:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941495.075983:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1773941495.075985:0:12219:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000040:2.0:1773941495.075989:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60a80 x1860112241159936/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941505 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941495.075991:0:12219:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941495.075995:0:12219:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941495.075997:0:12219:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941495.076001:0:12219:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941495.076003:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941495.076004:0:12219:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941495.076007:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941495.076009:0:12219:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941495.076010:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce6c00. 00000001:00000001:0.0:1773941495.076012:0:12219:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941495.076014:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941495.076015:0:12219:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:2.0:1773941495.076017:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941495.076018:0:12219:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941495.076021:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 10000000:00000001:0.0:1773941495.076022:0:12219:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941495.076026:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941495.076027:0:12219:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:2.0:1773941495.076030:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737400. 00000100:00080000:0.0:1773941495.076033:0:12219:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:2.0:1773941495.076034:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941495.076035:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941495.076036:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60a80. 00000100:00000001:2.0:1773941495.076038:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941495.076039:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941495.076040:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941495.076043:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941495.076044:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941495.076048:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941495.076050:0:12219:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941495.076051:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941495.076053:0:12219:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941495.076056:0:12219:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941495.076058:0:12219:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941495.076060:0:12219:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941495.076061:0:12219:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941495.076063:0:12219:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941495.076074:0:12219:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941495.076075:0:12219:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941495.076076:0:12219:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941495.076079:0:12219:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941495.076081:0:12219:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941495.076083:0:12219:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941495.076084:0:12219:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941495.076086:0:12219:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941495.076088:0:12219:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941495.079749:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890eca00. 00010000:00000010:1.1:1773941495.079759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000400:00000010:3.0:1773941495.199824:0:2349:0:(lib-lnet.h:485:lnet_rspt_free()) free 'rspt': 56 at ffff8b5b817f4ea8. 00000400:00000010:3.0:1773941495.199840:0:2349:0:(lib-lnet.h:485:lnet_rspt_free()) free 'rspt': 56 at ffff8b5b8a087268. 00000020:00000001:0.0:1773941495.647986:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941495.647991:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941495.648011:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941496.607849:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941496.607858:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941496.607863:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941496.607866:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941496.607867:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941496.607869:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941496.607872:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941496.607874:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941496.607877:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941496.607880:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941496.607882:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941496.607894:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941496.607899:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941496.607904:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941496.607907:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941496.607926:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941496.607934:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941496.607942:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941496.607947:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941496.607950:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941496.607958:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890eca00. 00010000:00000001:1.0:1773941496.607964:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941496.607968:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941496.607972:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941496.607975:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1da9 to hash 00000020:00000001:1.0:1773941496.607978:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941496.607979:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941496.607983:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941496.607987:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941496.607990:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941496.607995:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941496.608007:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941496.608020:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941496.608028:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941496.608030:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941496.608036:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941496.608039:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941496.608041:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941496.608043:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000001:1.0:1773941496.608048:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941496.608053:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941496.608055:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941496.608057:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941496.608072:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241160064/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941496.608083:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941496.608085:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941496.608089:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941496.608091:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941496.608094:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941496.608098:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241160064:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941496.608103:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941496.608108:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941496.608110:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941496.608113:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941496.608117:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734c00. 02000000:00000001:1.0:1773941496.608120:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941496.608128:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941496.608132:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941496.608139:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941496.608146:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241160064, portal 25 00000100:00000001:1.0:1773941496.608148:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941496.608150:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:1.0:1773941496.608156:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241160064/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941512 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941496.608164:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941496.608167:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941496.608170:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1f68. 00000100:00000200:1.0:1773941496.608174:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241160064, offset 0 00000400:00000200:1.0:1773941496.608181:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941496.608189:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941496.608196:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941496.608201:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941496.608206:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941496.608210:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941496.608218:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941496.608222:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[124, 123] 00000400:00000200:1.0:1773941496.608226:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941496.608231:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [230:230:256:4294967295] 192.168.204.153@tcp LPNI seq info [125:125:8:4294967295] 00000400:00000200:1.0:1773941496.608236:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941496.608247:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941496.608254:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941496.608260:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b400. 00000800:00000200:1.0:1773941496.608267:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941496.608275:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941496.608279:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b400 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941496.608298:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941496.608302:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941496.608305:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294936971 00000100:00000001:1.0:1773941496.608313:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294935437 00000100:00000001:1.0:1773941496.608319:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294936971 00000100:00000001:1.0:1773941496.608324:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294935437 00000100:00000001:1.0:1773941496.608328:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941496.608332:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941496.608335:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941496.608337:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941496.608340:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941496.608343:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941496.608346:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941496.608422:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941496.608427:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b400. 00000400:00000200:3.0:1773941496.608432:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941496.608437:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941496.608441:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1f68 00000400:00000010:3.0:1773941496.608443:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1f68. 00000100:00000001:3.0:1773941496.608446:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941496.608451:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241160064/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941512 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941496.608462:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941496.608466:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d63800 x1860112241160064/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941512 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941496.608475:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941496.608477:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941496.609117:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941496.609129:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941496.609132:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941496.609135:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941496.609141:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941496.609147:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941496.609154:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941496.609158:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941496.609162:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941496.609167:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321606f80 00000400:00000200:2.0:1773941496.609179:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941496.609184:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x1105 [1] + 224 00000800:00000001:2.0:1773941496.609192:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941496.609201:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941496.609204:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941496.609208:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941496.609212:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941496.609219:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241160064/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941512 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941496.609230:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d63800 x1860112241160064/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941512 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941496.609255:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941496.609263:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941496.609267:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941496.609285:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941496.609292:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941496.609294:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941496.609296:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941496.609300:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941496.609302:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941496.609305:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941496.609316:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241160064/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941512 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941496.609328:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241160064/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941512 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941496.609336:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941496.609339:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941496.609342:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941496.609344:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941496.609347:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941496.609349:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941496.609352:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941496.609357:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=75 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941496.609361:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941496.609364:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941496.609369:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941496.609372:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941496.609376:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941496.609379:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941496.609383:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941496.609387:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941496.609391:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941496.609393:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941496.609395:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941496.609397:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941496.609402:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d63800 x1860112241160064/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941512 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941496.609411:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941496.609412:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941496.609414:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941496.609417:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941496.609420:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941496.609422:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941496.609429:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941496.609430:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941496.609431:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941496.609433:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941496.609435:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941496.609438:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941496.609440:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941496.609443:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941496.609446:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 265 00000400:00000200:1.0:1773941496.609451:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941496.609456:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941496.609460:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941496.609463:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941496.609466:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941496.609469:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941496.609472:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941496.609474:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941496.609476:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941496.609479:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941496.609481:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941496.609484:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941496.609487:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941496.609490:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941496.609492:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941496.609494:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941496.609498:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941496.609502:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941496.609505:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941496.609508:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941496.609511:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941496.609523:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941496.609526:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941496.609533:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941496.609537:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60700. 00000020:00000040:1.0:1773941496.609540:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941496.609544:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941496.609546:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941496.609548:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941496.609550:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941496.609552:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941496.609556:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734800. 00000100:00000001:1.0:1773941496.609560:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941496.609566:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60700 x1860112241160192/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941496.609575:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941496.609578:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941496.609583:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006bd2135a] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60700 x1860112241160192/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941496.609602:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941496.609605:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941496.609607:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941496.609609:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941496.609612:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941496.609613:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941496.609615:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941496.609618:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941496.609620:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941496.609626:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241160064/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941512 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941496.609635:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941496.609636:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941496.609639:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941496.609642:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941496.609643:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941496.609646:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941496.609647:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941496.609655:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60700 x1860112241160192/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941496.609657:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241160064/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941512 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941496.609686:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941496.609688:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941496.609691:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941496.609694:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1773941496.609730:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241160064:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941496.609737:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941496.609740:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941496.609742:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941496.609743:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941496.609746:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1773941496.609752:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241160192:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000040:1.0:1773941496.609754:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241160064/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941512 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941496.609761:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941496.609767:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941496.609769:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941496.609771:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941496.609774:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 02000000:00000001:2.0:1773941496.609775:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941496.609780:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000010:2.0:1773941496.609783:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce6c00. 00000100:00000001:1.0:1773941496.609783:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941496.609789:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941496.609790:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941496.609794:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941496.609798:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:1.0:1773941496.609798:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941496.609802:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 00000020:00000001:1.0:1773941496.609804:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941496.609809:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000400:00000010:2.0:1773941496.609812:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9898. 00010000:00010000:1.0:1773941496.609815:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941496.609818:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34770. 00000100:00000200:2.0:1773941496.609825:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241160192, portal 25 00000100:00000001:2.0:1773941496.609844:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941496.609848:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121071360 : -128249588480256 : ffff8b5b90d60700) 00010000:00000001:1.0:1773941496.609848:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941496.609856:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941496.609861:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60700 x1860112241160192/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941506 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941496.609875:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:2.0:1773941496.609880:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:1.0:1773941496.609880:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941496.609888:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941496.609895:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34990. 00010000:00000001:1.0:1773941496.609895:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000200:2.0:1773941496.609900:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241160192, offset 0 00010000:00000001:1.0:1773941496.609904:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941496.609909:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941496.609910:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da9 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941496.609918:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941496.609926:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941496.609927:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:1.0:1773941496.609930:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941496.609932:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941496.609935:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941496.609938:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00010000:1.0:1773941496.609941:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da9 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941496.609944:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941496.609950:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941496.609958:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[124, 125] 00010000:00010000:1.0:1773941496.609958:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941496.609963:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941496.609967:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 10000000:00010000:1.0:1773941496.609968:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da9 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941496.609974:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [231:231:256:4294967295] 192.168.204.143@tcp LPNI seq info [126:126:8:4294967295] 00010000:00000001:1.0:1773941496.609986:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:2.0:1773941496.609987:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941496.609991:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941496.609994:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941496.609998:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000400:00000200:2.0:1773941496.610004:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000020:00000001:1.0:1773941496.610004:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941496.610009:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000800:00000200:2.0:1773941496.610015:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941496.610015:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941496.610021:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da9 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:2.0:1773941496.610023:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0300. 00000800:00000200:2.0:1773941496.610032:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00000001:1.0:1773941496.610036:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941496.610040:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da9 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941496.610044:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941496.610050:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0300 type 1, nob 616 niov 1 nkiov 1 10000000:01000000:1.0:1773941496.610056:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941496.610060:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941496.610062:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941496.610065:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:2.0:1773941496.610067:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941496.610068:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941496.610074:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941496.610078:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:2.0:1773941496.610079:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294936973 00010000:00000001:1.0:1773941496.610082:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941496.610085:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941496.610088:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941496.610093:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1da9 from hash 00000100:00000001:2.0:1773941496.610094:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294936971 00010000:00000001:1.0:1773941496.610098:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:2.0:1773941496.610101:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294936973 00010000:00000001:1.0:1773941496.610102:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941496.610105:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941496.610109:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:2.0:1773941496.610110:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294936971 00010000:00000001:1.0:1773941496.610111:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941496.610116:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:2.0:1773941496.610117:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941496.610121:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941496.610123:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941496.610124:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941496.610129:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941496.610129:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941496.610132:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941496.610134:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:1.0:1773941496.610137:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da9 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941496.610144:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941496.610146:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941496.610148:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:1.0:1773941496.610149:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:3.0:1773941496.610152:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941496.610153:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:2.0:1773941496.610155:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1773941496.610157:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:2.0:1773941496.610159:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000800:00000010:3.0:1773941496.610160:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0300. 00010000:00000001:1.0:1773941496.610160:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941496.610164:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941496.610164:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941496.610168:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941496.610169:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941496.610170:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941496.610172:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1da9 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941496.610182:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941496.610188:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34990 00010000:00000040:1.0:1773941496.610191:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890eca00 count: 0 00000400:00000010:3.0:1773941496.610193:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34990. 00000100:00000001:3.0:1773941496.610199:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941496.610206:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941496.610210:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941496.610213:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941496.610218:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241160064/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941512 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941496.610226:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941496.610229:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941496.610237:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734c00. 02000000:00000001:1.0:1773941496.610240:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941496.610242:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941496.610244:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941496.610246:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941496.610249:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941496.610252:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941496.610254:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941496.610256:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:1.0:1773941496.610259:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941496.610260:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941496.610262:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941496.610264:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941496.610267:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941496.610271:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941496.610275:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941496.610281:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941496.610285:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941496.610291:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000100:00000200:3.0:1773941496.610292:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60700 x1860112241160192/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941506 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941496.610298:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941496.610305:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00000100:00000001:3.0:1773941496.610306:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941496.610311:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60700 x1860112241160192/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941506 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00040000:00000010:1.0:1773941496.610313:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941496.610321:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 00000100:00000001:3.0:1773941496.610322:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:1.0:1773941496.610328:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000100:00000001:3.0:1773941496.610333:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000001:1.0:1773941496.610334:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941496.610340:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941496.610345:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941496.610349:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941496.610354:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941496.610357:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941496.610362:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941496.610365:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941496.610368:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941496.610371:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941496.610373:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941496.610382:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941496.610385:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941496.610387:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941496.610388:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941496.610390:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941496.610392:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941496.611716:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941496.611722:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941496.611724:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941496.611726:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941496.611729:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941496.611735:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941496.611739:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941496.611742:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941496.611745:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941496.611748:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607000 00000400:00000200:2.0:1773941496.611750:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941496.611753:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x110d [1] + 0 00000800:00000001:2.0:1773941496.611756:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941496.611761:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941496.611763:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941496.611765:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941496.611769:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941496.611773:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60700 x1860112241160192/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941506 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941496.611780:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60700 x1860112241160192/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941506 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941496.611799:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941496.611804:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941496.611806:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941496.611840:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941496.611845:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941496.611849:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941496.611851:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941496.611853:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9898. 00000400:00000200:0.0:1773941496.611857:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34770 00000400:00000010:0.0:1773941496.611858:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34770. 00000100:00000001:0.0:1773941496.611860:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941496.611865:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60700 x1860112241160192/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941506 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941496.611873:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60700 x1860112241160192/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941506 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941496.611878:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941496.611879:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941496.611881:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941496.611883:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941496.611893:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941496.611894:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941496.611898:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941496.611902:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=75 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941496.611905:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941496.611908:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941496.611910:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941496.611912:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941496.611914:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941496.611917:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941496.611923:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60700 x1860112241160192/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941506 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941496.611928:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941496.611930:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941496.611931:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941496.611933:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941496.611936:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941496.611938:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941496.611939:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941496.611940:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941496.611942:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941496.611944:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941496.611945:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941496.611948:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941496 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941496.611950:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941496.611955:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941496.611956:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941496.611958:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941496.611959:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941496.611960:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941496.611963:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941496.611965:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941496.611966:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941496.611969:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941496.611976:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941496.611978:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941496.612233:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941496.612238:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941496.612246:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60700 x1860112241160192/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941506 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941496.612258:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241160192:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941496.612263:0:12220:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941496.612264:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941496.612268:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941496.612273:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60700 x1860112241160192/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941506 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941496.612275:0:12220:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941496.612280:0:12220:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941496.612283:0:12220:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941496.612286:0:12220:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941496.612290:0:12220:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941496.612291:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941496.612293:0:12220:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941496.612294:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941496.612297:0:12220:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941496.612299:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce6c00. 00000001:00000001:0.0:1773941496.612300:0:12220:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941496.612304:0:12220:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941496.612305:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941496.612306:0:12220:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941496.612309:0:2351:0:(genops.c:1236:class_import_put()) Process entered 10000000:00000001:0.0:1773941496.612310:0:12220:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941496.612312:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941496.612313:0:12220:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:2.0:1773941496.612318:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:0.0:1773941496.612319:0:12220:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:2.0:1773941496.612321:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734800. 00000100:00000001:0.0:1773941496.612323:0:12220:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:2.0:1773941496.612327:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1773941496.612328:0:12220:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:2.0:1773941496.612331:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941496.612332:0:12220:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941496.612335:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60700. 00000100:00080000:0.0:1773941496.612336:0:12220:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941496.612341:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941496.612341:0:12220:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941496.612342:0:12220:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:2.0:1773941496.612345:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000040:0.0:1773941496.612345:0:12220:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:2.0:1773941496.612350:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941496.612355:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941496.612357:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941496.612360:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1773941496.612360:0:12220:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:2.0:1773941496.612362:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941496.612363:0:12220:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941496.612367:0:12220:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941496.612371:0:12220:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941496.612376:0:12220:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941496.612379:0:12220:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941496.612381:0:12220:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941496.612384:0:12220:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941496.612386:0:12220:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941496.615732:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890eca00. 00010000:00000010:1.1:1773941496.615740:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:3.0:1773941496.671888:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941496.671896:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941496.671901:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1773941497.695931:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941497.695941:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941497.695946:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941498.399892:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941498.399903:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941498.399909:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941498.399920:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941498.399923:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941498.399925:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941498.399928:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941498.399931:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941498.399934:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941498.399937:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941498.399939:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941498.399955:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941498.399961:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941498.399967:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941498.399972:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941498.399979:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941498.399989:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941498.399999:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.400004:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941498.400008:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941498.400018:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890eca00. 00010000:00000001:1.0:1773941498.400026:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941498.400030:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941498.400036:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941498.400040:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1db0 to hash 00000020:00000001:1.0:1773941498.400043:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941498.400045:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941498.400048:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941498.400054:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941498.400076:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941498.400081:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941498.400091:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941498.400103:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941498.400108:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941498.400110:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941498.400116:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941498.400118:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941498.400121:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941498.400122:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000001:1.0:1773941498.400132:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941498.400136:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.400138:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941498.400140:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941498.400152:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241160320/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.400160:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941498.400162:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.400166:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941498.400168:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941498.400170:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941498.400173:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241160320:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941498.400176:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941498.400183:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941498.400185:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.400187:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941498.400191:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734c00. 02000000:00000001:1.0:1773941498.400196:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941498.400203:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941498.400207:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941498.400214:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941498.400222:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241160320, portal 25 00000100:00000001:1.0:1773941498.400223:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941498.400224:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:1.0:1773941498.400229:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241160320/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941514 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.400235:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941498.400237:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941498.400239:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1220. 00000100:00000200:1.0:1773941498.400242:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241160320, offset 0 00000400:00000200:1.0:1773941498.400248:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941498.400256:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941498.400264:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941498.400269:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941498.400273:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941498.400276:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941498.400278:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941498.400281:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[126, 125] 00000400:00000200:1.0:1773941498.400284:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941498.400288:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [232:232:256:4294967295] 192.168.204.153@tcp LPNI seq info [127:127:8:4294967295] 00000400:00000200:1.0:1773941498.400291:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941498.400300:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941498.400305:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941498.400309:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024bf00. 00000800:00000200:1.0:1773941498.400316:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941498.400323:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941498.400328:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024bf00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941498.400344:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941498.400354:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941498.400356:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294938763 00000100:00000001:1.0:1773941498.400363:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294936973 00000100:00000001:1.0:1773941498.400369:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294938763 00000100:00000001:1.0:1773941498.400373:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294936973 00000100:00000001:1.0:1773941498.400376:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941498.400378:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.400380:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941498.400382:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941498.400384:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941498.400387:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941498.400390:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941498.400497:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941498.400504:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024bf00. 00000400:00000200:3.0:1773941498.400511:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941498.400518:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941498.400523:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1220 00000400:00000010:3.0:1773941498.400527:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1220. 00000100:00000001:3.0:1773941498.400531:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941498.400537:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241160320/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941514 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941498.400550:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941498.400554:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d63800 x1860112241160320/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941514 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941498.400563:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.400565:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941498.401394:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.401414:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941498.401434:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.401439:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941498.401447:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941498.401454:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941498.401461:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941498.401465:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941498.401470:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.401473:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607080 00000400:00000200:2.0:1773941498.401476:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.401481:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x111d [1] + 224 00000800:00000001:2.0:1773941498.401488:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.401499:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941498.401501:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941498.401505:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941498.401510:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941498.401517:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241160320/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941514 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941498.401526:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d63800 x1860112241160320/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.401547:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941498.401555:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.401558:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.401591:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941498.401599:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941498.401602:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941498.401604:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941498.401609:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941498.401611:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941498.401614:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941498.401621:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241160320/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941498.401634:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241160320/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.401642:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941498.401644:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.401647:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941498.401650:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941498.401654:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941498.401656:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.401660:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941498.401667:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=77 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941498.401672:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941498.401675:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941498.401680:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941498.401691:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941498.401696:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941498.401700:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941498.401727:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941498.401733:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941498.401737:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941498.401740:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941498.401742:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941498.401744:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941498.401750:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d63800 x1860112241160320/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.401759:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941498.401761:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941498.401763:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941498.401766:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941498.401770:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941498.401771:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.401774:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941498.401775:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941498.401777:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941498.401778:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.401780:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941498.401783:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941498.401785:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941498.401789:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941498.401792:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 267 00000400:00000200:1.0:1773941498.401814:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941498.401820:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941498.401825:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941498.401828:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941498.401832:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941498.401834:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941498.401843:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941498.401846:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941498.401849:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941498.401852:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941498.401855:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941498.401857:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941498.401861:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941498.401864:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941498.401866:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941498.401868:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941498.401871:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941498.401876:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941498.401879:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.401882:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941498.401886:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941498.401889:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.401892:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941498.401895:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941498.401899:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63b80. 00000020:00000040:1.0:1773941498.401903:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941498.401907:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941498.401909:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.401912:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941498.401923:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941498.401925:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941498.401931:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0735400. 00000100:00000001:1.0:1773941498.401936:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941498.401942:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63b80 x1860112241160448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941498.401951:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941498.401954:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941498.401960:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000957145d0] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63b80 x1860112241160448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.401989:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.401992:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941498.401994:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941498.401997:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.402000:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941498.402002:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941498.402004:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941498.402006:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941498.402009:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941498.402016:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241160320/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941514 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.402025:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941498.402028:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.402030:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.402033:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941498.402037:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941498.402038:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.402043:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941498.402050:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241160320/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941514 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941498.402056:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63b80 x1860112241160448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941498.402085:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241160320:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941498.402088:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941498.402093:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941498.402100:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941498.402101:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941498.402103:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941498.402108:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941498.402109:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.402114:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941498.402117:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941498.402124:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241160320/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941514 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941498.402131:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241160448:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941498.402149:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941498.402154:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941498.402157:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.402160:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941498.402162:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:1.0:1773941498.402170:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 02000000:00000010:2.0:1773941498.402171:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce6c00. 00000100:00000001:1.0:1773941498.402177:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:2.0:1773941498.402179:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.402182:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941498.402190:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941498.402191:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000010:2.0:1773941498.402195:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9e70. 00000400:00000010:2.0:1773941498.402202:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34880. 00010000:00000001:1.0:1773941498.402205:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941498.402209:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:2.0:1773941498.402213:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241160448, portal 25 00000020:00000040:1.0:1773941498.402213:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 00000100:00000001:2.0:1773941498.402222:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000020:00000001:1.0:1773941498.402223:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000001:2.0:1773941498.402227:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121084800 : -128249588466816 : ffff8b5b90d63b80) 00010000:00000001:1.0:1773941498.402230:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00010000:1.0:1773941498.402238:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941498.402240:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63b80 x1860112241160448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.402278:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941498.402279:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:2.0:1773941498.402285:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941498.402294:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34bb0. 00010000:00010000:1.0:1773941498.402295:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941498.402300:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241160448, offset 0 00000400:00000200:2.0:1773941498.402315:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941498.402315:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941498.402322:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941498.402332:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941498.402340:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941498.402343:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941498.402345:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00010000:1.0:1773941498.402349:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db0 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941498.402354:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941498.402364:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941498.402365:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941498.402370:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941498.402375:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941498.402378:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941498.402384:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:1.0:1773941498.402385:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db0 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941498.402399:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[126, 127] 00010000:00010000:1.0:1773941498.402408:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:2.0:1773941498.402411:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 10000000:00000001:1.0:1773941498.402415:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941498.402422:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [233:233:256:4294967295] 192.168.204.143@tcp LPNI seq info [128:128:8:4294967295] 10000000:00010000:1.0:1773941498.402422:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db0 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941498.402447:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.402470:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941498.402476:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941498.402478:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941498.402480:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941498.402482:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000020:00000001:1.0:1773941498.402491:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941498.402496:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000800:00000200:2.0:1773941498.402503:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941498.402505:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941498.402511:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db0 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:2.0:1773941498.402512:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0500. 10000000:00000001:1.0:1773941498.402529:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:2.0:1773941498.402530:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00010000:1.0:1773941498.402537:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db0 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941498.402546:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 10000000:01000000:1.0:1773941498.402551:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000800:00000200:2.0:1773941498.402554:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0500 type 1, nob 616 niov 1 nkiov 1 10000000:00000040:1.0:1773941498.402559:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941498.402563:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941498.402568:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941498.402570:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941498.402574:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941498.402576:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:2.0:1773941498.402577:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.402580:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941498.402584:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941498.402587:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000040:1.0:1773941498.402591:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1db0 from hash 00000100:00000001:2.0:1773941498.402595:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294938765 00010000:00000001:1.0:1773941498.402597:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941498.402602:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941498.402604:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941498.402605:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:2.0:1773941498.402607:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294938763 00010000:00000001:1.0:1773941498.402609:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941498.402613:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:2.0:1773941498.402616:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294938765 00010000:00000001:1.0:1773941498.402617:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941498.402622:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941498.402626:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.402627:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294938763 00000100:00000001:2.0:1773941498.402638:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:00000001:1.0:1773941498.402638:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.402647:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941498.402647:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db0 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941498.402651:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941498.402656:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1773941498.402660:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0500. 00000100:00000001:2.0:1773941498.402666:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941498.402669:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941498.402670:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:1.0:1773941498.402673:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941498.402676:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:2.0:1773941498.402678:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941498.402681:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000400:00000200:3.0:1773941498.402682:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941498.402684:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:1.0:1773941498.402686:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941498.402690:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34bb0 00010000:00000001:1.0:1773941498.402693:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941498.402695:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1773941498.402696:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34bb0. 00010000:00000001:1.0:1773941498.402697:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941498.402744:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db0 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941498.402755:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890eca00 count: 0 00000100:00000001:2.0:1773941498.402764:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941498.402768:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.402772:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941498.402774:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.402775:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941498.402776:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941498.402783:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000200:3.0:1773941498.402786:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63b80 x1860112241160448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941498.402793:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241160320/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941498.402824:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941498.402828:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000040:3.0:1773941498.402832:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63b80 x1860112241160448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941508 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941498.402834:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941498.402840:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734c00. 00000100:00000001:3.0:1773941498.402847:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.402847:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941498.402852:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1773941498.402853:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941498.402856:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941498.402859:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941498.402863:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941498.402867:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941498.402870:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941498.402872:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:1.0:1773941498.402876:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941498.402879:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941498.402882:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941498.402885:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941498.402888:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941498.402901:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941498.402906:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941498.402911:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941498.402915:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941498.402920:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941498.402924:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941498.402928:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941498.402933:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941498.402936:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941498.402941:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941498.402943:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941498.402947:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941498.402951:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941498.402953:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941498.402956:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941498.402959:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941498.402963:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941498.402965:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941498.402967:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941498.402969:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941498.402972:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941498.402974:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941498.402976:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941498.402977:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941498.402978:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941498.402979:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941498.402981:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941498.404071:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.404080:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941498.404082:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.404084:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941498.404088:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.404092:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941498.404096:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.404099:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.404102:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.404104:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607100 00000400:00000200:2.0:1773941498.404107:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.404116:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1125 [1] + 0 00000800:00000001:2.0:1773941498.404121:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.404127:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941498.404130:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941498.404133:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941498.404136:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941498.404140:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63b80 x1860112241160448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941508 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941498.404148:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63b80 x1860112241160448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.404166:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941498.404171:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.404174:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.404207:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941498.404219:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941498.404224:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941498.404226:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941498.404229:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9e70. 00000400:00000200:0.0:1773941498.404234:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34880 00000400:00000010:0.0:1773941498.404236:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34880. 00000100:00000001:0.0:1773941498.404240:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941498.404248:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63b80 x1860112241160448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941498.404257:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63b80 x1860112241160448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.404264:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941498.404265:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.404267:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941498.404269:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941498.404270:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941498.404271:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.404274:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941498.404277:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=77 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941498.404280:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941498.404282:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941498.404283:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.404285:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941498.404287:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.404289:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941498.404293:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63b80 x1860112241160448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.404300:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941498.404303:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941498.404305:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.404307:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941498.404313:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941498.404315:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941498.404317:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941498.404319:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.404327:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941498.404330:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941498.404333:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941498.404337:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941498 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941498.404351:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941498.404358:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941498.404361:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941498.404363:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941498.404365:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.404367:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941498.404371:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941498.404373:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941498.404375:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941498.404379:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941498.404382:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941498.404384:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941498.404629:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.404633:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941498.404639:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63b80 x1860112241160448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941508 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941498.404648:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241160448:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941498.404655:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.404657:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941498.404660:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63b80 x1860112241160448/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941508 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.404663:0:12221:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941498.404666:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941498.404669:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941498.404672:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce6c00. 02000000:00000001:2.0:1773941498.404676:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941498.404678:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00080000:0.0:1773941498.404680:0:12221:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000020:00000040:2.0:1773941498.404681:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941498.404686:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941498.404689:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0735400. 00000100:00100000:0.0:1773941498.404689:0:12221:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:2.0:1773941498.404697:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1773941498.404697:0:12221:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941498.404732:0:12221:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941498.404737:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:0.0:1773941498.404739:0:12221:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000010:2.0:1773941498.404740:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63b80. 00000100:00000001:2.0:1773941498.404747:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:0.0:1773941498.404747:0:12221:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941498.404749:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.404753:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:0.0:1773941498.404755:0:12221:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.404757:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941498.404764:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941498.404764:0:12221:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941498.404770:0:12221:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941498.404772:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000001:00000001:0.0:1773941498.404778:0:12221:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.404779:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941498.404794:0:12221:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.404810:0:12221:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941498.404816:0:12221:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941498.404819:0:12221:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941498.404822:0:12221:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941498.404825:0:12221:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941498.404828:0:12221:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941498.404839:0:12221:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941498.404841:0:12221:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941498.404843:0:12221:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941498.404861:0:12221:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941498.404863:0:12221:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.404865:0:12221:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941498.404871:0:12221:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941498.404876:0:12221:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.404879:0:12221:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941498.404882:0:12221:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941498.404885:0:12221:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941498.404887:0:12221:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941498.408732:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890eca00. 00010000:00000010:1.1:1773941498.408739:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:3.0:1773941498.719895:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941498.719927:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941498.719934:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941498.783873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1773941498.783882:0:36:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1773941498.783892:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941498.783897:0:36:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941498.783901:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941498.783904:0:36:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941498.783924:0:36:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941498.783930:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b9024b800. 00000100:00000001:1.0:1773941498.783936:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941498.783946:0:36:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS req@ffff8b5b90d63800 x1860112241160576/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941498.783955:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941498.783958:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941498.783963:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc928f9c] to pc [ptlrpcd_00_00+0] req@ffff8b5b90d63800 x1860112241160576/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.783990:0:36:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941498.783994:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (274) 00000100:00000001:1.0:1773941498.784009:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1773941498.784015:0:2353:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:1.0:1773941498.784018:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.784028:0:2353:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941498.784032:0:2353:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.784034:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.784036:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.784039:0:2353:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941498.784043:0:2353:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1773941498.784043:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941498.784053:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241160576/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.784063:0:2353:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941498.784065:0:2353:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.784069:0:2353:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941498.784073:0:2353:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941498.784076:0:2353:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941498.784079:0:2353:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2353:1860112241160576:192.168.204.143@tcp:400:kworker.0 00000100:00000001:1.0:1773941498.784082:0:2353:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941498.784087:0:2353:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941498.784089:0:2353:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.784091:0:2353:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941498.784095:0:2353:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941498.784096:0:2353:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941498.784102:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941498.784111:0:2353:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941498.784126:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941498.784135:0:2353:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241160576, portal 25 00000100:00000001:1.0:1773941498.784137:0:2353:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941498.784138:0:2353:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:1.0:1773941498.784143:0:2353:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241160576/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941514 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.784148:0:2353:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941498.784151:0:2353:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941498.784153:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1198. 00000100:00000200:1.0:1773941498.784155:0:2353:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241160576, offset 0 00000400:00000200:1.0:1773941498.784160:0:2353:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941498.784166:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941498.784172:0:2353:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941498.784177:0:2353:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941498.784180:0:2353:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941498.784184:0:2353:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941498.784186:0:2353:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941498.784188:0:2353:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[128, 127] 00000400:00000200:1.0:1773941498.784190:0:2353:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941498.784194:0:2353:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [234:234:256:4294967295] 192.168.204.153@tcp LPNI seq info [129:129:8:4294967295] 00000400:00000200:1.0:1773941498.784196:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941498.784204:0:2353:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941498.784208:0:2353:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941498.784211:0:2353:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b100. 00000800:00000200:1.0:1773941498.784217:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941498.784223:0:2353:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941498.784227:0:2353:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b100 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941498.784239:0:2353:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941498.784242:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941498.784244:0:2353:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294939147 00000100:00000001:1.0:1773941498.784254:0:2353:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294938765 00000100:00000001:1.0:1773941498.784258:0:2353:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294939147 00000100:00000001:1.0:1773941498.784261:0:2353:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294938765 00000100:00000001:1.0:1773941498.784264:0:2353:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941498.784266:0:2353:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.784270:0:2353:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.784272:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941498.784411:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941498.784419:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b100. 00000400:00000200:3.0:1773941498.784428:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941498.784438:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941498.784446:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1198 00000400:00000010:3.0:1773941498.784450:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1198. 00000100:00000001:3.0:1773941498.784455:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941498.784464:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241160576/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941514 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941498.784477:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941498.784482:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241160576/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941514 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941498.784491:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.784493:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941498.785057:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.785072:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941498.785076:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.785079:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941498.785085:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941498.785091:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941498.785100:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941498.785104:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941498.785108:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.785112:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607180 00000400:00000200:2.0:1773941498.785118:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.785124:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x1135 [1] + 224 00000800:00000001:2.0:1773941498.785132:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.785145:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941498.785150:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941498.785154:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941498.785161:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941498.785168:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241160576/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941498.785180:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b90d63800 x1860112241160576/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.785197:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941498.785206:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.785210:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.785224:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.785227:0:2353:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941498.785232:0:2353:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941498.785233:0:2353:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941498.785236:0:2353:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941498.785239:0:2353:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941498.785241:0:2353:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941498.785243:0:2353:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941498.785248:0:2353:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241160576/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941498.785256:0:2353:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241160576/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.785261:0:2353:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941498.785262:0:2353:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.785264:0:2353:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941498.785266:0:2353:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941498.785268:0:2353:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941498.785269:0:2353:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.785271:0:2353:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941498.785276:0:2353:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=77 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941498.785278:0:2353:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941498.785280:0:2353:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941498.785282:0:2353:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941498.785284:0:2353:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941498.785286:0:2353:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941498.785290:0:2353:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941498.785299:0:2353:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941498.785302:0:2353:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941498.785304:0:2353:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941498.785305:0:2353:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941498.785307:0:2353:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941498.785309:0:2353:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d63800 x1860112241160576/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.785314:0:2353:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941498.785315:0:2353:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941498.785316:0:2353:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941498.785319:0:2353:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941498.785321:0:2353:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941498.785322:0:2353:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.785324:0:2353:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941498.785324:0:2353:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941498.785325:0:2353:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941498.785326:0:2353:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.785327:0:2353:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941498.785329:0:2353:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941498.785330:0:2353:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941498.785332:0:2353:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941498.785334:0:2353:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 268 00000400:00000200:1.0:1773941498.785337:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941498.785340:0:2353:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941498.785343:0:2353:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941498.785346:0:2353:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941498.785347:0:2353:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941498.785349:0:2353:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941498.785351:0:2353:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941498.785353:0:2353:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941498.785355:0:2353:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941498.785357:0:2353:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941498.785358:0:2353:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941498.785363:0:2353:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941498.785365:0:2353:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941498.785367:0:2353:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941498.785368:0:2353:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941498.785369:0:2353:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941498.785371:0:2353:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941498.785374:0:2353:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941498.785375:0:2353:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.785377:0:2353:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941498.785379:0:2353:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941498.785381:0:2353:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.785382:0:2353:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941498.785383:0:2353:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941498.785386:0:2353:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61c00. 00000020:00000040:1.0:1773941498.785388:0:2353:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941498.785390:0:2353:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941498.785391:0:2353:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.785393:0:2353:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941498.785394:0:2353:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941498.785396:0:2353:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941498.785399:0:2353:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734c00. 00000100:00000001:1.0:1773941498.785402:0:2353:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941498.785406:0:2353:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d61c00 x1860112241160704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941498.785412:0:2353:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941498.785414:0:2353:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941498.785416:0:2353:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbfa947b] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d61c00 x1860112241160704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.785431:0:2353:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.785433:0:2353:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941498.785434:0:2353:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:1.0:1773941498.785438:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241160576/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.785446:0:2353:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941498.785449:0:2353:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941498.785451:0:2353:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941498.785455:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241160576/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941514 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941498.785461:0:2353:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2353:1860112241160576:192.168.204.143@tcp:400:kworker.0 00000100:00000001:2.0:1773941498.785463:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.785465:0:2353:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.785467:0:2353:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941498.785469:0:2353:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241160576/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941514 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.785475:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941498.785476:0:2353:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941498.785479:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1773941498.785479:0:2353:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941498.785484:0:2353:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941498.785487:0:2353:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941498.785489:0:2353:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000040:2.0:1773941498.785490:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d61c00 x1860112241160704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1773941498.785491:0:2353:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941498.785495:0:2353:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941498.785498:0:2353:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b9024b800. 02000000:00000001:1.0:1773941498.785504:0:2353:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941498.785507:0:2353:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941498.785508:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000010:1.0:1773941498.785509:0:2353:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:2.0:1773941498.785511:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.785513:0:2353:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941498.785516:0:2353:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941498.785517:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941498.785518:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941498.785522:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941498.785523:0:2353:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941498.785525:0:2353:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941498.785528:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.785528:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.785531:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941498.785535:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241160704:192.168.204.143@tcp:250:ptlrpcd_00_01.0 00000100:00000001:2.0:1773941498.785541:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941498.785544:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941498.785546:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941498.785548:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941498.785553:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce6c00. 02000000:00000001:2.0:1773941498.785555:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941498.785559:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941498.785564:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9370. 00000400:00000010:2.0:1773941498.785568:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34ee0. 00000100:00000200:2.0:1773941498.785572:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241160704, portal 25 00000100:00000001:2.0:1773941498.785574:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941498.785576:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121076736 : -128249588474880 : ffff8b5b90d61c00) 00000100:00000040:2.0:1773941498.785581:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d61c00 x1860112241160704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.785589:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941498.785592:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941498.785594:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34660. 00000100:00000200:2.0:1773941498.785597:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241160704, offset 0 00000400:00000200:2.0:1773941498.785601:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941498.785605:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.785609:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941498.785612:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941498.785615:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941498.785623:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941498.785625:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941498.785629:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[128, 129] 00000400:00000200:2.0:1773941498.785633:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:2.0:1773941498.785636:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [235:235:256:4294967295] 192.168.204.143@tcp LPNI seq info [130:130:8:4294967295] 00000400:00000200:2.0:1773941498.785641:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.785649:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:2.0:1773941498.785654:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:2.0:1773941498.785658:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0600. 00000800:00000200:2.0:1773941498.785663:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:2.0:1773941498.785671:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:2.0:1773941498.785674:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0600 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941498.785690:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941498.785695:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941498.785697:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294939148 00000100:00000001:2.0:1773941498.785701:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294939147 00000100:00000001:2.0:1773941498.785729:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294939149 00000100:00000001:2.0:1773941498.785734:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294939147 00000100:00000001:2.0:1773941498.785738:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941498.785741:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.785744:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.785746:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.785751:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941498.785753:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941498.785755:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941498.785757:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941498.785759:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.785762:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941498.785824:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941498.785828:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0600. 00000400:00000200:3.0:1773941498.785832:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941498.785836:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941498.785839:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34660 00000400:00000010:3.0:1773941498.785840:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34660. 00000100:00000001:3.0:1773941498.785843:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941498.785847:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61c00 x1860112241160704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941498.785854:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941498.785856:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d61c00 x1860112241160704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941508 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941498.785861:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.785862:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941498.787307:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.787315:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941498.787319:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.787321:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941498.787333:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.787338:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941498.787345:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.787348:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.787351:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.787355:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607200 00000400:00000200:2.0:1773941498.787359:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941498.787364:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x113d [1] + 0 00000800:00000001:2.0:1773941498.787369:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.787377:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941498.787380:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941498.787384:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941498.787388:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941498.787394:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61c00 x1860112241160704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941508 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941498.787406:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d61c00 x1860112241160704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.787428:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941498.787433:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941498.787436:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.787465:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941498.787468:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941498.787474:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941498.787475:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941498.787478:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9370. 00000400:00000200:0.0:1773941498.787482:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34ee0 00000400:00000010:0.0:1773941498.787484:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34ee0. 00000100:00000001:0.0:1773941498.787487:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941498.787495:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61c00 x1860112241160704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941498.787505:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61c00 x1860112241160704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.787510:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941498.787512:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.787514:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941498.787516:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941498.787518:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941498.787520:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.787522:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941498.787527:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=77 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941498.787531:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941498.787533:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941498.787547:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.787551:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941498.787553:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.787557:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941498.787563:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d61c00 x1860112241160704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.787572:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941498.787575:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941498.787577:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.787580:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941498.787587:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941498.787591:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941498.787593:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941498.787596:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.787599:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941498.787602:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941498.787605:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941498.787609:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941498 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941498.787613:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941498.787621:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941498.787623:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941498.787625:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941498.787627:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.787629:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941498.787632:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941498.787635:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941498.787637:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941498.787640:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941498.787643:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941498.787646:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941498.787946:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.787952:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941498.787961:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d61c00 x1860112241160704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941508 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941498.787972:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d61c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241160704:192.168.204.143@tcp:250:ptlrpcd_00_01.0 00000100:00000001:2.0:1773941498.787980:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.787983:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941498.787989:0:12222:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941498.787990:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d61c00 x1860112241160704/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941508 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941498.788003:0:12222:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:2.0:1773941498.788005:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:0.0:1773941498.788008:0:12222:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:2.0:1773941498.788018:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1773941498.788018:0:12222:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941498.788023:0:12222:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000010:2.0:1773941498.788026:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce6c00. 00000001:00000001:0.0:1773941498.788028:0:12222:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941498.788033:0:12222:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941498.788036:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941498.788038:0:12222:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941498.788042:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941498.788042:0:12222:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941498.788045:0:12222:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:2.0:1773941498.788049:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941498.788049:0:12222:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941498.788057:0:12222:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941498.788059:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941498.788063:0:12222:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:2.0:1773941498.788064:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734c00. 00000100:00080000:0.0:1773941498.788072:0:12222:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:2.0:1773941498.788073:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941498.788077:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941498.788080:0:12222:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000010:2.0:1773941498.788082:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61c00. 00000100:00000001:0.0:1773941498.788085:0:12222:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941498.788090:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941498.788090:0:12222:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.788095:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941498.788097:0:12222:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941498.788103:0:12222:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941498.788105:0:12222:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941498.788108:0:12222:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:2.0:1773941498.788111:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.788124:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941498.788126:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.788129:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941498.788132:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941498.788132:0:12222:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941498.788136:0:12222:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.788139:0:12222:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941498.788143:0:12222:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941498.788146:0:12222:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.788148:0:12222:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941498.788150:0:12222:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941498.788153:0:12222:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941498.788155:0:12222:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941499.743906:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941499.743917:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941499.743923:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941500.063865:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941500.063873:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941500.063877:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941500.063880:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941500.063899:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941500.063900:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941500.063903:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941500.063904:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941500.063906:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941500.063908:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941500.063910:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941500.063916:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941500.063920:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941500.063924:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941500.063927:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941500.063932:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941500.063938:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941500.063944:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941500.063948:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941500.063951:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941500.063959:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890eca00. 00010000:00000001:1.0:1773941500.063964:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941500.063968:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941500.063972:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941500.063975:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1db7 to hash 00000020:00000001:1.0:1773941500.063978:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941500.063979:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941500.063982:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941500.063986:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941500.063990:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941500.063995:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941500.064025:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941500.064037:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941500.064050:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941500.064052:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941500.064056:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941500.064058:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941500.064060:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941500.064061:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000001:1.0:1773941500.064065:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941500.064069:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941500.064071:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941500.064072:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941500.064080:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241160832/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941500.064087:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941500.064088:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941500.064091:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941500.064093:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941500.064095:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941500.064097:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241160832:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941500.064101:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941500.064105:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941500.064107:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941500.064109:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941500.064113:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0737000. 02000000:00000001:1.0:1773941500.064114:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941500.064121:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941500.064124:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941500.064129:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941500.064135:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241160832, portal 25 00000100:00000001:1.0:1773941500.064138:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941500.064139:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:1.0:1773941500.064144:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241160832/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941516 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941500.064164:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941500.064167:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941500.064170:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f17f8. 00000100:00000200:1.0:1773941500.064175:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241160832, offset 0 00000400:00000200:1.0:1773941500.064182:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941500.064189:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941500.064195:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941500.064199:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941500.064203:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941500.064206:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941500.064208:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941500.064211:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[130, 129] 00000400:00000200:1.0:1773941500.064213:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941500.064216:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [236:236:256:4294967295] 192.168.204.153@tcp LPNI seq info [131:131:8:4294967295] 00000400:00000200:1.0:1773941500.064219:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941500.064226:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941500.064230:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941500.064234:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b800. 00000800:00000200:1.0:1773941500.064240:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941500.064246:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941500.064250:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b800 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941500.064265:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941500.064268:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941500.064270:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294940427 00000100:00000001:1.0:1773941500.064276:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294939148 00000100:00000001:1.0:1773941500.064280:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294940427 00000100:00000001:1.0:1773941500.064283:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294939149 00000100:00000001:1.0:1773941500.064286:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941500.064288:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941500.064295:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941500.064296:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941500.064298:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941500.064303:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941500.064306:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941500.064435:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941500.064445:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b800. 00000400:00000200:3.0:1773941500.064454:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941500.064475:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941500.064481:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f17f8 00000400:00000010:3.0:1773941500.064484:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f17f8. 00000100:00000001:3.0:1773941500.064488:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941500.064495:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241160832/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941516 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941500.064506:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941500.064509:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d63800 x1860112241160832/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941516 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941500.064515:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941500.064517:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941500.065359:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941500.065374:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941500.065378:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941500.065381:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941500.065389:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941500.065397:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941500.065405:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941500.065411:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941500.065418:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941500.065423:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607280 00000400:00000200:2.0:1773941500.065428:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941500.065436:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x114d [1] + 224 00000800:00000001:2.0:1773941500.065445:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941500.065456:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941500.065459:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941500.065463:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941500.065468:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941500.065474:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241160832/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941516 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941500.065502:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d63800 x1860112241160832/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941516 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941500.065524:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941500.065533:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941500.065537:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941500.065573:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941500.065580:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941500.065583:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941500.065586:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941500.065592:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941500.065595:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941500.065598:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941500.065605:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241160832/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941516 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941500.065616:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241160832/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941516 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941500.065624:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941500.065626:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941500.065628:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941500.065631:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941500.065634:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941500.065636:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941500.065639:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941500.065645:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=79 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941500.065648:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941500.065651:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941500.065655:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941500.065658:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941500.065661:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941500.065664:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941500.065668:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941500.065673:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941500.065679:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941500.065690:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941500.065692:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941500.065694:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941500.065698:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d63800 x1860112241160832/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941516 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941500.065735:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941500.065737:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941500.065739:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941500.065742:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941500.065745:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941500.065746:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941500.065749:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941500.065750:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941500.065751:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941500.065753:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941500.065755:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941500.065757:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941500.065759:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941500.065762:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941500.065780:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 269 00000400:00000200:1.0:1773941500.065787:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941500.065792:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941500.065797:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941500.065800:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941500.065802:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941500.065804:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941500.065808:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941500.065810:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941500.065812:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941500.065815:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941500.065817:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941500.065820:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941500.065823:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941500.065833:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941500.065835:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941500.065837:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941500.065840:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941500.065844:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941500.065847:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941500.065850:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941500.065854:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941500.065856:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941500.065859:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941500.065861:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941500.065866:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d61500. 00000020:00000040:1.0:1773941500.065869:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941500.065872:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941500.065874:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941500.065877:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941500.065879:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941500.065881:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941500.065886:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734000. 00000100:00000001:1.0:1773941500.065890:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941500.065897:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d61500 x1860112241160960/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941500.065905:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941500.065908:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941500.065913:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000bdbed3f] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d61500 x1860112241160960/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941500.065937:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941500.065940:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941500.065942:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941500.065945:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941500.065948:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941500.065950:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941500.065958:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941500.065963:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941500.065966:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941500.065973:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241160832/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941516 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941500.065975:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941500.065981:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941500.065984:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941500.065987:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941500.065991:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941500.065998:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:2.0:1773941500.065999:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d61500 x1860112241160960/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941500.066003:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941500.066017:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241160832/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941516 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941500.066020:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941500.066025:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941500.066035:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941500.066042:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1773941500.066044:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241160832:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941500.066048:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941500.066055:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941500.066061:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941500.066064:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00100000:2.0:1773941500.066067:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241160960:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941500.066067:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941500.066074:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241160832/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941516 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941500.066077:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941500.066084:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941500.066087:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941500.066089:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941500.066093:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 02000000:00000001:2.0:1773941500.066095:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941500.066100:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000010:2.0:1773941500.066104:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce6c00. 00000100:00000001:1.0:1773941500.066105:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941500.066112:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941500.066114:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941500.066120:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941500.066125:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:1.0:1773941500.066127:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:2.0:1773941500.066134:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9268. 00000020:00000040:1.0:1773941500.066134:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 00000020:00000001:1.0:1773941500.066145:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941500.066149:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000400:00000010:2.0:1773941500.066152:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f347f8. 00010000:00010000:1.0:1773941500.066155:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941500.066163:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241160960, portal 25 00000100:00000001:2.0:1773941500.066167:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941500.066169:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121074944 : -128249588476672 : ffff8b5b90d61500) 00000100:00000040:2.0:1773941500.066179:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d61500 x1860112241160960/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941510 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941500.066204:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941500.066206:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:1.0:1773941500.066213:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941500.066225:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941500.066232:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34f68. 00010000:00000001:1.0:1773941500.066234:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000200:2.0:1773941500.066240:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241160960, offset 0 00010000:00010000:1.0:1773941500.066240:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941500.066257:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941500.066259:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941500.066262:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db7 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941500.066264:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941500.066275:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941500.066277:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941500.066280:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941500.066286:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941500.066288:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00010000:1.0:1773941500.066292:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db7 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941500.066297:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941500.066306:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00010000:1.0:1773941500.066308:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:2.0:1773941500.066315:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000001:1.0:1773941500.066316:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941500.066322:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db7 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941500.066323:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941500.066332:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[130, 131] 00000400:00000200:2.0:1773941500.066348:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941500.066350:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941500.066354:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941500.066359:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [237:237:256:4294967295] 192.168.204.143@tcp LPNI seq info [132:132:8:4294967295] 00000020:00000001:1.0:1773941500.066359:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941500.066365:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000020:00000001:1.0:1773941500.066373:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000400:00000200:2.0:1773941500.066380:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941500.066382:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941500.066398:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:2.0:1773941500.066399:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00010000:1.0:1773941500.066406:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db7 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941500.066414:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 10000000:00000001:1.0:1773941500.066428:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000010:2.0:1773941500.066429:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0e00. 10000000:00010000:1.0:1773941500.066433:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db7 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941500.066446:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:01000000:1.0:1773941500.066448:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941500.066454:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941500.066456:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941500.066459:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000800:00000200:2.0:1773941500.066461:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941500.066463:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000800:00000200:2.0:1773941500.066468:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0e00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941500.066469:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941500.066473:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941500.066478:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941500.066480:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941500.066484:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1db7 from hash 00010000:00000001:1.0:1773941500.066487:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941500.066489:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941500.066491:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941500.066492:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941500.066496:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941500.066500:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:2.0:1773941500.066505:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941500.066505:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:2.0:1773941500.066513:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294940429 00010000:00000001:1.0:1773941500.066515:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941500.066519:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941500.066523:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941500.066525:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941500.066527:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294940427 00010000:00010000:1.0:1773941500.066530:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db7 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941500.066536:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294940429 00010000:00000001:1.0:1773941500.066542:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941500.066545:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294940427 00010000:00000001:1.0:1773941500.066546:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941500.066553:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:2.0:1773941500.066555:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941500.066556:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:2.0:1773941500.066562:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941500.066565:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941500.066566:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941500.066569:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941500.066574:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:3.0:1773941500.066576:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0e00. 00000100:00000001:2.0:1773941500.066576:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941500.066581:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941500.066585:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00010000:1.0:1773941500.066587:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1db7 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941500.066596:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941500.066604:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:3.0:1773941500.066606:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34f68 00000100:00000001:2.0:1773941500.066610:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000010:3.0:1773941500.066611:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34f68. 00000100:00000001:2.0:1773941500.066622:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000040:1.0:1773941500.066624:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890eca00 count: 0 00000100:00000001:2.0:1773941500.066627:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941500.066631:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941500.066635:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941500.066646:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941500.066650:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941500.066656:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941500.066663:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241160832/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941516 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941500.066675:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1773941500.066678:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941500.066683:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000200:3.0:1773941500.066687:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d61500 x1860112241160960/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941510 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1773941500.066688:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0737000. 02000000:00000001:1.0:1773941500.066699:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941500.066725:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941500.066730:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d61500 x1860112241160960/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941510 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941500.066738:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941500.066739:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1773941500.066753:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941500.066757:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941500.066762:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941500.066780:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941500.066787:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941500.066790:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941500.066792:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:1.0:1773941500.066796:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941500.066798:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941500.066801:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941500.066805:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941500.066808:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941500.066812:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941500.066816:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941500.066822:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941500.066826:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941500.066831:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941500.066836:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941500.066840:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941500.066851:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941500.066856:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 10000000:00000040:1.0:1773941500.066861:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941500.066864:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941500.066869:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941500.066873:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941500.066876:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941500.066880:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941500.066884:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941500.066889:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941500.066892:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941500.066896:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941500.066899:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941500.066901:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941500.066904:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941500.066907:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941500.066909:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941500.066911:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941500.066913:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941500.066915:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941500.068125:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941500.068133:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941500.068136:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941500.068138:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941500.068141:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941500.068145:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941500.068150:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941500.068152:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941500.068156:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941500.068158:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607300 00000400:00000200:2.0:1773941500.068162:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941500.068166:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1155 [1] + 0 00000800:00000001:2.0:1773941500.068170:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941500.068175:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941500.068177:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941500.068179:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941500.068183:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941500.068187:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d61500 x1860112241160960/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941510 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941500.068195:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d61500 x1860112241160960/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941510 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941500.068212:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941500.068216:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941500.068219:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941500.068262:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941500.068267:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941500.068273:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941500.068275:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941500.068279:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9268. 00000400:00000200:0.0:1773941500.068285:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f347f8 00000400:00000010:0.0:1773941500.068288:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f347f8. 00000100:00000001:0.0:1773941500.068299:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941500.068306:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d61500 x1860112241160960/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941510 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941500.068319:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d61500 x1860112241160960/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941510 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941500.068328:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941500.068330:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941500.068334:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941500.068337:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941500.068339:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941500.068342:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941500.068345:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941500.068352:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=79 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941500.068357:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941500.068360:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941500.068362:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941500.068366:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941500.068368:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941500.068372:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941500.068377:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d61500 x1860112241160960/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941510 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941500.068387:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941500.068390:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941500.068392:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941500.068395:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941500.068403:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941500.068407:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941500.068410:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941500.068413:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941500.068417:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941500.068419:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941500.068423:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941500.068427:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941500 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941500.068438:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941500.068447:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941500.068450:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941500.068454:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941500.068457:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941500.068459:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941500.068464:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941500.068467:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941500.068470:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941500.068475:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941500.068479:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941500.068482:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941500.068797:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941500.068801:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941500.068807:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d61500 x1860112241160960/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941510 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941500.068821:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d61500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241160960:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941500.068821:0:12223:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941500.068828:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941500.068830:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941500.068833:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d61500 x1860112241160960/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941510 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941500.068836:0:12223:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941500.068846:0:12223:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941500.068847:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941500.068851:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1773941500.068852:0:12223:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:2.0:1773941500.068855:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce6c00. 10000000:01000000:0.0:1773941500.068856:0:12223:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941500.068859:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941500.068862:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941500.068863:0:12223:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000040:2.0:1773941500.068865:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941500.068869:0:12223:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:2.0:1773941500.068870:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941500.068876:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734000. 00000001:00000001:0.0:1773941500.068878:0:12223:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941500.068880:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:0.0:1773941500.068882:0:12223:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941500.068885:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:0.0:1773941500.068886:0:12223:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000010:2.0:1773941500.068890:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d61500. 00000001:00000001:0.0:1773941500.068891:0:12223:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941500.068896:0:12223:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941500.068897:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941500.068900:0:12223:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941500.068901:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941500.068906:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941500.068907:0:12223:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941500.068912:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941500.068913:0:12223:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941500.068917:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941500.068918:0:12223:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941500.068923:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941500.068924:0:12223:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941500.068926:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941500.068931:0:12223:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941500.068936:0:12223:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941500.068938:0:12223:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941500.068941:0:12223:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941500.068963:0:12223:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941500.068966:0:12223:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941500.068974:0:12223:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941500.068979:0:12223:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941500.068986:0:12223:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941500.068989:0:12223:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941500.068991:0:12223:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941500.068995:0:12223:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941500.068998:0:12223:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941500.071749:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890eca00. 00010000:00000010:1.1:1773941500.071762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:3.0:1773941500.767956:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941500.767967:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941500.767973:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1773941501.791887:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941501.791894:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941501.791900:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941501.919821:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941501.919828:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941501.919832:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941501.919834:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941501.919835:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941501.919836:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941501.919839:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941501.919841:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941501.919843:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941501.919845:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941501.919846:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941501.919855:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941501.919859:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941501.919863:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941501.919865:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941501.919883:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941501.919889:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941501.919895:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941501.919898:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941501.919901:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941501.919907:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890eca00. 00010000:00000001:1.0:1773941501.919911:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941501.919914:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941501.919917:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941501.919919:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1dbe to hash 00000020:00000001:1.0:1773941501.919920:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941501.919921:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941501.919924:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941501.919927:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941501.919929:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941501.919932:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dbe lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941501.919940:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dbe lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941501.919950:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dbe lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941501.919955:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941501.919956:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941501.919961:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941501.919963:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941501.919965:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941501.919966:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000001:1.0:1773941501.919970:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941501.919973:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941501.919975:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941501.919976:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941501.919988:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241161088/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941501.919995:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941501.919997:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941501.920000:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941501.920002:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941501.920004:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941501.920006:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241161088:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941501.920008:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941501.920012:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941501.920014:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941501.920016:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941501.920020:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0736800. 02000000:00000001:1.0:1773941501.920021:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941501.920028:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941501.920030:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941501.920035:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941501.920040:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241161088, portal 25 00000100:00000001:1.0:1773941501.920042:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941501.920043:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:1.0:1773941501.920047:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241161088/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941517 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941501.920052:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941501.920054:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941501.920057:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f17f8. 00000100:00000200:1.0:1773941501.920059:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241161088, offset 0 00000400:00000200:1.0:1773941501.920064:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941501.920070:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941501.920077:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941501.920081:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941501.920084:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941501.920087:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941501.920093:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941501.920095:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[132, 131] 00000400:00000200:1.0:1773941501.920098:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941501.920101:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [238:238:256:4294967295] 192.168.204.153@tcp LPNI seq info [133:133:8:4294967295] 00000400:00000200:1.0:1773941501.920104:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941501.920112:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941501.920127:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941501.920132:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b800. 00000800:00000200:1.0:1773941501.920137:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941501.920143:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941501.920146:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b800 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941501.920159:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941501.920162:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941501.920164:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294942283 00000100:00000001:1.0:1773941501.920170:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294940429 00000100:00000001:1.0:1773941501.920175:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294942283 00000100:00000001:1.0:1773941501.920177:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294940429 00000100:00000001:1.0:1773941501.920180:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941501.920182:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941501.920184:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941501.920185:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941501.920187:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941501.920189:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941501.920192:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941501.920280:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941501.920285:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b800. 00000400:00000200:3.0:1773941501.920290:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941501.920296:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941501.920300:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f17f8 00000400:00000010:3.0:1773941501.920308:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f17f8. 00000100:00000001:3.0:1773941501.920311:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941501.920315:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241161088/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941517 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941501.920323:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941501.920326:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d63800 x1860112241161088/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941517 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941501.920332:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941501.920333:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941501.921247:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941501.921257:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941501.921259:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941501.921260:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941501.921264:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941501.921268:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941501.921273:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941501.921275:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941501.921278:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941501.921281:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607380 00000400:00000200:2.0:1773941501.921290:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941501.921294:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x1165 [1] + 224 00000800:00000001:2.0:1773941501.921299:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941501.921305:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941501.921306:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941501.921309:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941501.921313:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941501.921317:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241161088/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941517 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941501.921325:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d63800 x1860112241161088/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941517 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941501.921341:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941501.921349:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941501.921352:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941501.921379:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941501.921385:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941501.921386:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941501.921388:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941501.921392:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941501.921393:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941501.921395:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941501.921407:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241161088/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941517 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941501.921415:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241161088/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941517 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941501.921420:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941501.921421:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941501.921423:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941501.921425:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941501.921427:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941501.921428:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941501.921430:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941501.921434:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=80 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941501.921436:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941501.921439:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941501.921442:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941501.921444:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941501.921446:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941501.921448:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941501.921452:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941501.921455:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941501.921457:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941501.921458:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941501.921460:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941501.921461:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941501.921464:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d63800 x1860112241161088/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941517 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941501.921469:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941501.921470:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941501.921471:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941501.921473:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941501.921475:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941501.921476:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941501.921481:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941501.921482:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941501.921483:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941501.921484:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941501.921485:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941501.921487:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941501.921488:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941501.921490:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941501.921492:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 270 00000400:00000200:1.0:1773941501.921495:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941501.921498:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941501.921500:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941501.921503:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941501.921504:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941501.921506:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941501.921508:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941501.921509:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941501.921511:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941501.921513:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941501.921514:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941501.921516:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941501.921518:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941501.921519:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941501.921520:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941501.921521:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941501.921523:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941501.921525:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941501.921527:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941501.921529:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941501.921531:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941501.921533:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941501.921534:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941501.921538:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941501.921541:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60e00. 00000020:00000040:1.0:1773941501.921543:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941501.921546:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941501.921547:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941501.921549:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941501.921550:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941501.921552:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941501.921555:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737000. 00000100:00000001:1.0:1773941501.921558:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941501.921562:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60e00 x1860112241161216/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941501.921568:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941501.921570:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941501.921573:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5e4d003] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60e00 x1860112241161216/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941501.921589:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941501.921591:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941501.921593:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941501.921595:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941501.921596:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941501.921597:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941501.921599:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941501.921601:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941501.921602:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941501.921606:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241161088/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941517 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941501.921612:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941501.921613:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941501.921614:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941501.921618:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941501.921618:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941501.921622:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941501.921623:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941501.921631:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241161088/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941517 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941501.921632:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60e00 x1860112241161216/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941501.921648:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941501.921651:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941501.921656:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941501.921658:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941501.921663:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941501.921663:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241161088:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941501.921670:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941501.921671:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241161216:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941501.921674:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941501.921677:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941501.921678:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:2.0:1773941501.921681:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941501.921682:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941501.921685:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941501.921689:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1773941501.921689:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241161088/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941517 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1773941501.921693:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce6c00. 00000100:00000001:1.0:1773941501.921697:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941501.921698:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941501.921722:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000010:1.0:1773941501.921725:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941501.921729:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941501.921744:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000010:2.0:1773941501.921747:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9bb0. 00010000:00000001:1.0:1773941501.921747:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941501.921751:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941501.921753:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:2.0:1773941501.921754:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34550. 00000020:00000040:1.0:1773941501.921755:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 00000100:00000200:2.0:1773941501.921760:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241161216, portal 25 00000020:00000001:1.0:1773941501.921760:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941501.921763:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000001:2.0:1773941501.921764:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941501.921768:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00010000:00010000:1.0:1773941501.921769:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dbe lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941501.921778:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60e00 x1860112241161216/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941511 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941501.921779:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941501.921786:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dbe lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941501.921791:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941501.921796:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941501.921797:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000010:2.0:1773941501.921799:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34d48. 00010000:00010000:1.0:1773941501.921801:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dbe lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941501.921802:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241161216, offset 0 00010000:00000001:1.0:1773941501.921810:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941501.921811:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941501.921815:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941501.921819:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941501.921822:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00010000:1.0:1773941501.921823:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dbe lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941501.921827:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941501.921833:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941501.921833:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941501.921837:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941501.921837:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941501.921842:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941501.921843:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941501.921846:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dbe lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941501.921847:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[132, 133] 00000400:00000200:2.0:1773941501.921859:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00010000:1.0:1773941501.921860:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:2.0:1773941501.921864:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [239:239:256:4294967295] 192.168.204.143@tcp LPNI seq info [134:134:8:4294967295] 10000000:00000001:1.0:1773941501.921865:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941501.921868:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dbe lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941501.921873:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941501.921879:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941501.921882:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941501.921885:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941501.921887:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000020:00000040:1.0:1773941501.921888:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000020:00000001:1.0:1773941501.921893:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941501.921896:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000800:00000200:2.0:1773941501.921897:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941501.921899:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000010:2.0:1773941501.921902:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0200. 00010000:00010000:1.0:1773941501.921903:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dbe lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941501.921912:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00000001:1.0:1773941501.921912:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941501.921916:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dbe lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941501.921922:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000800:00000200:2.0:1773941501.921923:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 10000000:00000040:1.0:1773941501.921926:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000800:00000200:2.0:1773941501.921928:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0200 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:1.0:1773941501.921929:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941501.921935:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941501.921936:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:2.0:1773941501.921942:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941501.921942:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941501.921946:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:2.0:1773941501.921949:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941501.921949:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941501.921953:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941501.921954:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294942285 00000020:00000040:1.0:1773941501.921956:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1dbe from hash 00010000:00000001:1.0:1773941501.921959:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941501.921960:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941501.921963:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941501.921966:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:2.0:1773941501.921967:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294942283 00010000:00000001:1.0:1773941501.921968:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941501.921971:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:2.0:1773941501.921973:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294942285 00010000:00000001:1.0:1773941501.921974:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941501.921976:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941501.921978:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941501.921979:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294942283 10000000:00000001:1.0:1773941501.921980:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941501.921986:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00010000:1.0:1773941501.921987:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dbe lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941501.921992:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941501.921997:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941501.921998:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941501.921999:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941501.922002:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941501.922003:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941501.922006:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941501.922008:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:3.0:1773941501.922009:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941501.922009:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941501.922012:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000010:3.0:1773941501.922014:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0200. 00010000:00010000:1.0:1773941501.922015:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dbe lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941501.922017:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941501.922021:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941501.922023:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1773941501.922024:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941501.922025:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941501.922028:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:1.0:1773941501.922031:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890eca00 count: 0 00000100:00000001:2.0:1773941501.922032:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941501.922033:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941501.922039:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34d48 00000400:00000010:3.0:1773941501.922042:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34d48. 00000100:00000001:3.0:1773941501.922045:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941501.922046:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941501.922049:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000200:3.0:1773941501.922053:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60e00 x1860112241161216/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941511 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941501.922053:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941501.922060:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241161088/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941517 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941501.922066:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941501.922068:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941501.922074:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0736800. 02000000:00000001:1.0:1773941501.922076:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941501.922078:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941501.922079:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941501.922081:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941501.922083:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941501.922086:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941501.922087:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941501.922088:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:1.0:1773941501.922090:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941501.922091:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941501.922092:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941501.922094:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941501.922097:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941501.922100:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941501.922103:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941501.922106:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941501.922110:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941501.922116:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000100:00000001:3.0:1773941501.922118:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000010:1.0:1773941501.922120:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 00000100:00000040:3.0:1773941501.922125:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60e00 x1860112241161216/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941511 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 80000000:00000010:1.0:1773941501.922126:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941501.922131:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 00000100:00000001:3.0:1773941501.922136:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 20000000:00000010:1.0:1773941501.922137:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 00000100:00000001:3.0:1773941501.922139:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000040:1.0:1773941501.922142:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941501.922145:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941501.922149:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941501.922152:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941501.922155:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941501.922158:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941501.922160:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941501.922164:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941501.922166:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941501.922168:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941501.922169:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941501.922171:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941501.922177:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941501.922179:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941501.922180:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941501.922181:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941501.922183:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941501.922184:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941501.923509:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941501.923517:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941501.923518:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941501.923520:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941501.923524:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941501.923528:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941501.923532:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941501.923534:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941501.923537:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941501.923539:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607400 00000400:00000200:2.0:1773941501.923542:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941501.923545:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x116d [1] + 0 00000800:00000001:2.0:1773941501.923548:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941501.923553:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941501.923554:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941501.923557:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941501.923560:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941501.923564:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60e00 x1860112241161216/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941511 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941501.923571:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60e00 x1860112241161216/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941511 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941501.923593:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941501.923598:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941501.923600:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941501.923626:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941501.923628:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941501.923632:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941501.923633:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941501.923635:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9bb0. 00000400:00000200:0.0:1773941501.923639:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34550 00000400:00000010:0.0:1773941501.923640:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34550. 00000100:00000001:0.0:1773941501.923642:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941501.923647:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60e00 x1860112241161216/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941511 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941501.923655:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60e00 x1860112241161216/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941511 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941501.923660:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941501.923662:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941501.923663:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941501.923664:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941501.923665:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941501.923667:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941501.923668:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941501.923670:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=80 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941501.923673:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941501.923675:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941501.923676:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941501.923678:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941501.923679:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941501.923686:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941501.923689:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60e00 x1860112241161216/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941511 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941501.923694:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941501.923696:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941501.923697:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941501.923699:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941501.923717:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941501.923719:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941501.923721:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941501.923723:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941501.923725:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941501.923727:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941501.923729:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941501.923742:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941501 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941501.923745:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941501.923750:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941501.923751:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941501.923753:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941501.923754:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941501.923756:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941501.923759:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941501.923760:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941501.923762:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941501.923764:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941501.923767:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941501.923769:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941501.923980:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941501.923984:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941501.923989:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60e00 x1860112241161216/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941511 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941501.923996:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241161216:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941501.924000:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941501.924001:0:12225:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:1.0:1773941501.924002:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941501.924007:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60e00 x1860112241161216/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941511 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1773941501.924012:0:12225:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:1.0:1773941501.924017:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:2.0:1773941501.924018:0:12225:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:1.0:1773941501.924019:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941501.924022:0:12225:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:1.0:1773941501.924023:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce6c00. 10000000:01000000:2.0:1773941501.924025:0:12225:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:1.0:1773941501.924027:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:2.0:1773941501.924029:0:12225:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:1.0:1773941501.924030:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:2.0:1773941501.924033:0:12225:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:1.0:1773941501.924034:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941501.924038:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:2.0:1773941501.924039:0:12225:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941501.924040:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737000. 00000001:00000001:2.0:1773941501.924045:0:12225:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:1.0:1773941501.924046:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:2.0:1773941501.924048:0:12225:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:1.0:1773941501.924049:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:2.0:1773941501.924051:0:12225:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941501.924052:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60e00. 10000000:00000001:2.0:1773941501.924057:0:12225:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941501.924057:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941501.924062:0:12225:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941501.924062:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941501.924065:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941501.924067:0:12225:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941501.924070:0:12225:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941501.924070:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941501.924073:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941501.924074:0:12225:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941501.924076:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941501.924077:0:12225:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941501.924079:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941501.924081:0:12225:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941501.924084:0:12225:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941501.924085:0:12225:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941501.924086:0:12225:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:2.0:1773941501.924105:0:12225:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941501.924107:0:12225:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941501.924108:0:12225:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941501.924110:0:12225:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:2.0:1773941501.924113:0:12225:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941501.924114:0:12225:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941501.924116:0:12225:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:2.0:1773941501.924117:0:12225:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941501.924118:0:12225:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941501.928750:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890eca00. 00010000:00000010:1.1:1773941501.928757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:3.0:1773941502.815928:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941502.815936:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941502.815941:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941503.007836:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941503.007864:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941503.007868:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941503.007870:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941503.007871:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941503.007875:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941503.007878:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941503.007880:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941503.007882:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941503.007884:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941503.007885:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941503.007892:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60e00. 00000020:00000040:1.0:1773941503.007896:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941503.007900:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941503.007903:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941503.007908:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941503.007916:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941503.007922:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941503.007927:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941503.007930:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941503.007938:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890eca00. 00010000:00000001:1.0:1773941503.007944:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941503.007948:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941503.007952:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941503.007955:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1dc5 to hash 00000020:00000001:1.0:1773941503.007957:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941503.007959:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941503.007962:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941503.007967:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941503.007970:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941503.007975:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dc5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941503.007984:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dc5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941503.008002:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dc5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941503.008006:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941503.008008:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941503.008014:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941503.008015:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941503.008017:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941503.008019:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00000100:00000001:1.0:1773941503.008023:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941503.008026:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.008028:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941503.008029:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941503.008037:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60e00 x1860112241161344/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941503.008044:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941503.008046:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941503.008048:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941503.008050:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941503.008052:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941503.008055:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241161344:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941503.008058:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941503.008062:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941503.008064:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941503.008066:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941503.008069:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0737000. 02000000:00000001:1.0:1773941503.008071:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941503.008077:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941503.008080:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941503.008085:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941503.008091:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241161344, portal 25 00000100:00000001:1.0:1773941503.008092:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941503.008096:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00000100:00000040:1.0:1773941503.008100:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60e00 x1860112241161344/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941519 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941503.008106:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941503.008108:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941503.008110:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1198. 00000100:00000200:1.0:1773941503.008112:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241161344, offset 0 00000400:00000200:1.0:1773941503.008117:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941503.008124:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941503.008130:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941503.008134:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941503.008138:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941503.008141:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941503.008143:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941503.008145:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[134, 133] 00000400:00000200:1.0:1773941503.008148:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941503.008151:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [240:240:256:4294967295] 192.168.204.153@tcp LPNI seq info [135:135:8:4294967295] 00000400:00000200:1.0:1773941503.008154:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941503.008162:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941503.008166:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941503.008170:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b100. 00000800:00000200:1.0:1773941503.008175:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941503.008182:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941503.008185:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b100 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941503.008204:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941503.008207:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941503.008209:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294943371 00000100:00000001:1.0:1773941503.008215:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294942285 00000100:00000001:1.0:1773941503.008221:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294943371 00000100:00000001:1.0:1773941503.008225:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294942285 00000100:00000001:1.0:1773941503.008233:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941503.008236:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.008239:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941503.008241:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941503.008244:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941503.008247:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941503.008250:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941503.008353:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941503.008359:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b100. 00000400:00000200:3.0:1773941503.008365:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941503.008372:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941503.008378:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1198 00000400:00000010:3.0:1773941503.008381:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1198. 00000100:00000001:3.0:1773941503.008385:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941503.008397:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60e00 x1860112241161344/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941519 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941503.008409:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941503.008413:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d60e00 x1860112241161344/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941519 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941503.008422:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.008424:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941503.009310:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.009325:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941503.009327:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.009329:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941503.009333:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941503.009337:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941503.009342:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941503.009345:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941503.009348:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941503.009351:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607480 00000400:00000200:2.0:1773941503.009355:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941503.009358:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x117d [1] + 224 00000800:00000001:2.0:1773941503.009363:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.009369:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941503.009371:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941503.009374:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941503.009378:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941503.009382:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60e00 x1860112241161344/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941519 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941503.009391:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d60e00 x1860112241161344/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941519 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941503.009404:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941503.009410:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.009412:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941503.009434:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941503.009439:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941503.009441:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941503.009443:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941503.009446:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941503.009447:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941503.009449:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941503.009453:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60e00 x1860112241161344/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941519 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941503.009460:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60e00 x1860112241161344/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941519 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941503.009465:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941503.009467:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941503.009468:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941503.009470:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941503.009472:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941503.009473:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941503.009475:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941503.009479:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=82 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941503.009482:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941503.009484:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941503.009487:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941503.009490:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941503.009492:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941503.009494:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941503.009497:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941503.009500:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941503.009507:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941503.009509:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941503.009510:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941503.009512:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941503.009514:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d60e00 x1860112241161344/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941519 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941503.009519:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941503.009520:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941503.009522:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941503.009524:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941503.009526:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941503.009527:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.009528:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941503.009529:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941503.009530:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941503.009531:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.009533:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941503.009535:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941503.009536:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941503.009540:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941503.009542:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 272 00000400:00000200:1.0:1773941503.009547:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941503.009552:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941503.009556:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941503.009559:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941503.009562:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941503.009564:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941503.009568:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941503.009570:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941503.009572:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941503.009575:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941503.009577:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941503.009579:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941503.009586:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941503.009588:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941503.009589:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941503.009590:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941503.009592:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941503.009595:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941503.009597:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941503.009598:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941503.009601:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941503.009603:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.009604:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941503.009606:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941503.009608:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941503.009610:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941503.009612:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941503.009613:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.009615:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941503.009616:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941503.009618:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941503.009621:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0736800. 00000100:00000001:1.0:1773941503.009624:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941503.009628:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63800 x1860112241161472/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941503.009633:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941503.009636:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941503.009639:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc928f9c] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63800 x1860112241161472/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941503.009658:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.009660:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941503.009662:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941503.009664:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941503.009665:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941503.009676:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941503.009678:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941503.009680:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941503.009681:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941503.009685:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60e00 x1860112241161344/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941519 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941503.009690:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941503.009692:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941503.009695:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941503.009696:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941503.009699:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60e00 x1860112241161344/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941519 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941503.009739:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941503.009746:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1773941503.009748:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241161344:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941503.009749:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941503.009757:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241161472/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941503.009758:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941503.009765:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941503.009770:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941503.009772:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941503.009778:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60e00 x1860112241161344/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941519 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941503.009784:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941503.009787:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.009793:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941503.009793:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941503.009795:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1773941503.009797:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 02000000:00000001:0.0:1773941503.009799:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.009802:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941503.009805:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:0.0:1773941503.009805:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241161472:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941503.009810:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:0.0:1773941503.009811:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941503.009814:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:0.0:1773941503.009815:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1773941503.009816:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:0.0:1773941503.009818:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941503.009819:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 02000000:00000001:0.0:1773941503.009821:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:1.0:1773941503.009824:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941503.009827:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 02000000:00000010:0.0:1773941503.009827:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0000. 02000000:00000001:0.0:1773941503.009832:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941503.009834:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dc5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941503.009836:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941503.009839:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941503.009844:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941503.009849:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241161472, portal 25 00010000:00000001:1.0:1773941503.009850:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941503.009853:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941503.009855:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:0.0:1773941503.009860:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241161472/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941503.009862:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dc5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941503.009869:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941503.009872:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941503.009875:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c4c8. 00010000:00000001:1.0:1773941503.009878:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000200:0.0:1773941503.009879:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241161472, offset 0 00010000:00010000:1.0:1773941503.009882:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dc5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941503.009887:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941503.009893:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941503.009894:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941503.009896:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941503.009899:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dc5 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941503.009900:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941503.009907:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941503.009912:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941503.009913:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:0.0:1773941503.009916:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941503.009917:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941503.009922:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941503.009923:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941503.009926:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dc5 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941503.009935:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:0.0:1773941503.009935:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[134, 135] 10000000:00000001:1.0:1773941503.009940:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941503.009943:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dc5 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941503.009943:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941503.009949:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [241:241:256:4294967295] 192.168.204.143@tcp LPNI seq info [136:136:8:4294967295] 00010000:00000001:1.0:1773941503.009953:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941503.009956:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:0.0:1773941503.009957:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:1.0:1773941503.009958:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941503.009961:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000020:00000001:1.0:1773941503.009965:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941503.009970:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000400:00000200:0.0:1773941503.009971:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941503.009975:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941503.009979:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00010000:00000001:1.0:1773941503.009980:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941503.009985:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dc5 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941503.009990:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00000001:1.0:1773941503.009997:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941503.009999:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dc5 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941503.010000:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941503.010005:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 10000000:01000000:1.0:1773941503.010013:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941503.010015:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941503.010016:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941503.010018:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941503.010019:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941503.010021:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:0.0:1773941503.010022:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941503.010023:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941503.010028:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941503.010030:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941503.010030:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000040:1.0:1773941503.010034:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1dc5 from hash 00000100:00000001:0.0:1773941503.010035:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294943373 00010000:00000001:1.0:1773941503.010039:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941503.010042:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941503.010042:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294943371 00010000:00000001:1.0:1773941503.010045:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941503.010047:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:0.0:1773941503.010048:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294943373 00010000:00000001:1.0:1773941503.010050:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941503.010053:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:0.0:1773941503.010054:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294943371 00010000:00000001:1.0:1773941503.010055:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941503.010057:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941503.010060:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941503.010061:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:00000001:1.0:1773941503.010062:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941503.010066:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941503.010069:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941503.010072:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:1.0:1773941503.010074:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dc5 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941503.010078:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941503.010080:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941503.010081:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:1.0:1773941503.010083:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:3.0:1773941503.010085:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1773941503.010085:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1773941503.010086:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:0.0:1773941503.010089:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941503.010090:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000800:00000010:3.0:1773941503.010091:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000100:00000001:0.0:1773941503.010092:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941503.010097:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1773941503.010097:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941503.010098:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941503.010101:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941503.010103:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941503.010105:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dc5 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941503.010111:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890eca00 count: 0 00000400:00000200:3.0:1773941503.010117:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941503.010122:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c4c8 00010000:00000001:1.0:1773941503.010123:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941503.010125:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941503.010127:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000010:3.0:1773941503.010132:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c4c8. 00000100:00000040:1.0:1773941503.010133:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60e00 x1860112241161344/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941519 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941503.010139:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1773941503.010141:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941503.010144:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941503.010146:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0737000. 00000100:00000200:3.0:1773941503.010149:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241161472/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941503.010150:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941503.010153:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941503.010155:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941503.010158:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941503.010160:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941503.010165:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941503.010167:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941503.010170:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60e00. 00000100:00000001:3.0:1773941503.010171:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941503.010173:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941503.010176:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941503.010177:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241161472/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941503.010178:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941503.010183:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941503.010187:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:3.0:1773941503.010191:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 10000000:00000010:1.0:1773941503.010192:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0980. 00000020:00000010:1.0:1773941503.010198:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941503.010202:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941503.010205:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941503.010208:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941503.010210:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941503.010214:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941503.010224:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941503.010230:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 00000100:00000001:3.0:1773941503.010235:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000040:1.0:1773941503.010235:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941503.010240:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941503.010247:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941503.010252:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941503.010256:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941503.010261:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941503.010264:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941503.010268:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941503.010271:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941503.010274:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941503.010278:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0980. 10000000:01000000:1.0:1773941503.010280:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941503.010282:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941503.010284:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941503.010285:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941503.010287:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941503.010288:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941503.010289:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941503.011637:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.011646:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941503.011648:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.011651:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941503.011656:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941503.011663:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941503.011669:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941503.011673:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941503.011677:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941503.011680:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607500 00000400:00000200:2.0:1773941503.011691:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941503.011694:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1185 [1] + 0 00000800:00000001:2.0:1773941503.011699:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.011737:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941503.011741:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941503.011746:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941503.011750:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941503.011755:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241161472/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941503.011766:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63800 x1860112241161472/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941503.011785:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941503.011796:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.011800:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941503.011815:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941503.011818:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941503.011821:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941503.011822:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941503.011823:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941503.011826:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941503.011832:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941503.011834:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941503.011838:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241161472/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941503.011844:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241161472/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941503.011849:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941503.011850:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941503.011852:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941503.011853:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941503.011854:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941503.011855:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941503.011858:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941503.011861:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=82 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941503.011865:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941503.011867:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941503.011869:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941503.011872:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941503.011874:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941503.011876:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941503.011881:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241161472/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941503.011889:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941503.011891:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941503.011893:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941503.011896:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941503.011900:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941503.011903:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941503.011904:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941503.011906:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941503.011908:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941503.011910:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941503.011912:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941503.011916:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941503 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941503.011926:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941503.011933:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941503.011935:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941503.011937:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941503.011939:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941503.011941:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941503.011944:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941503.011945:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941503.011947:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941503.011950:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941503.011952:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941503.011954:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941503.012155:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.012161:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941503.012168:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241161472/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941513 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941503.012173:0:12226:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:1.0:1773941503.012178:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241161472:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00080000:0.0:1773941503.012183:0:12226:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:1.0:1773941503.012185:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941503.012187:0:12226:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941503.012189:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941503.012190:0:12226:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941503.012193:0:12226:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000040:1.0:1773941503.012194:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241161472/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941513 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:0.0:1773941503.012196:0:12226:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941503.012200:0:12226:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941503.012203:0:12226:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.012205:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941503.012206:0:12226:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:1.0:1773941503.012208:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941503.012209:0:12226:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941503.012211:0:12226:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941503.012212:0:12226:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941503.012214:0:12226:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:1.0:1773941503.012218:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0000. 00000100:00080000:0.0:1773941503.012220:0:12226:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:1.0:1773941503.012222:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941503.012226:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941503.012226:0:12226:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000040:1.0:1773941503.012229:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941503.012231:0:12226:0:(recover.c:147:ptlrpc_resend()) Process entered 00000020:00000001:1.0:1773941503.012232:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941503.012236:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0736800. 00000100:00000001:0.0:1773941503.012236:0:12226:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941503.012238:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941503.012241:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:0.0:1773941503.012242:0:12226:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000010:1.0:1773941503.012243:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:1.0:1773941503.012248:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941503.012249:0:12226:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:1.0:1773941503.012250:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941503.012253:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:0.0:1773941503.012253:0:12226:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941503.012258:0:12226:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:1.0:1773941503.012260:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941503.012263:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.012265:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941503.012266:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941503.012280:0:12226:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941503.012282:0:12226:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941503.012289:0:12226:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941503.012294:0:12226:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941503.012299:0:12226:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941503.012301:0:12226:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941503.012303:0:12226:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941503.012304:0:12226:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941503.012306:0:12226:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941503.015757:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890eca00. 00010000:00000010:1.1:1773941503.015765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:0.0:1773941503.839839:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941503.839845:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941503.839848:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1773941503.903862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1773941503.903873:0:37:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1773941503.903885:0:37:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021ca80. 00000020:00000040:2.0:1773941503.903891:0:37:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941503.903895:0:37:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941503.903899:0:37:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941503.903904:0:37:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941503.903913:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b98bb0c00. 00000100:00000001:2.0:1773941503.903920:0:37:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941503.903932:0:37:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS req@ffff8b5b9021ca80 x1860112241161600/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941503.903945:0:37:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941503.903949:0:37:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941503.903957:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000005f3b654] to pc [ptlrpcd_01_01+1] req@ffff8b5b9021ca80 x1860112241161600/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941503.903985:0:37:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941503.903991:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (279) 00000100:00000001:3.0:1773941503.904017:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941503.904021:0:2355:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941503.904024:0:2355:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941503.904030:0:2355:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021ca80 x1860112241161600/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941503.904038:0:2355:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941503.904039:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941503.904040:0:2355:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941503.904046:0:2355:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1773941503.904046:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941503.904050:0:2355:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941503.904053:0:2355:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941503.904056:0:2355:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2355:1860112241161600:192.168.204.143@tcp:400:kworker.0 00000100:00000001:3.0:1773941503.904069:0:2355:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941503.904074:0:2355:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941503.904076:0:2355:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941503.904078:0:2355:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941503.904088:0:2355:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b839e2600. 02000000:00000001:3.0:1773941503.904089:0:2355:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941503.904096:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941503.904099:0:2355:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941503.904106:0:2355:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941503.904114:0:2355:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241161600, portal 25 00000100:00000001:3.0:1773941503.904117:0:2355:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941503.904118:0:2355:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109259392 : -128249600292224 : ffff8b5b9021ca80) 00000100:00000040:3.0:1773941503.904125:0:2355:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021ca80 x1860112241161600/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941503.904133:0:2355:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941503.904136:0:2355:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941503.904139:0:2355:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6acc0. 00000100:00000200:3.0:1773941503.904142:0:2355:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241161600, offset 0 00000400:00000200:3.0:1773941503.904150:0:2355:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941503.904158:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941503.904166:0:2355:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941503.904171:0:2355:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941503.904176:0:2355:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941503.904180:0:2355:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941503.904183:0:2355:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941503.904187:0:2355:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[136, 135] 00000400:00000200:3.0:1773941503.904191:0:2355:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941503.904195:0:2355:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [242:242:256:4294967295] 192.168.204.153@tcp LPNI seq info [137:137:8:4294967295] 00000400:00000200:3.0:1773941503.904199:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941503.904209:0:2355:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941503.904215:0:2355:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941503.904226:0:2355:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69600. 00000800:00000200:3.0:1773941503.904233:0:2355:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941503.904241:0:2355:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941503.904245:0:2355:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69600 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1773941503.904263:0:2355:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941503.904267:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941503.904270:0:2355:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294944267 00000100:00000001:3.0:1773941503.904277:0:2355:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294943373 00000100:00000001:3.0:1773941503.904283:0:2355:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294944267 00000100:00000001:3.0:1773941503.904287:0:2355:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294943373 00000100:00000001:3.0:1773941503.904291:0:2355:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941503.904294:0:2355:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.904297:0:2355:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.904300:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941503.904307:0:2355:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941503.904309:0:2355:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1773941503.904312:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941503.904313:0:2355:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941503.904316:0:2355:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.904318:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941503.904378:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941503.904383:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69600. 00000400:00000200:2.0:1773941503.904388:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941503.904394:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941503.904399:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6acc0 00000400:00000010:2.0:1773941503.904402:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6acc0. 00000100:00000001:2.0:1773941503.904406:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941503.904411:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021ca80 x1860112241161600/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941503.904421:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941503.904424:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021ca80 x1860112241161600/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941519 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941503.904433:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941503.904434:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941503.905476:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.905486:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941503.905489:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.905491:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941503.905497:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941503.905502:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941503.905509:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941503.905513:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941503.905517:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941503.905521:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607580 00000400:00000200:2.0:1773941503.905526:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941503.905531:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x118d [1] + 224 00000800:00000001:2.0:1773941503.905538:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.905545:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941503.905547:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941503.905551:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941503.905556:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941503.905560:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021ca80 x1860112241161600/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941519 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941503.905577:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b9021ca80 x1860112241161600/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941503.905592:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941503.905600:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.905604:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941503.905617:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941503.905621:0:2355:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941503.905625:0:2355:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941503.905627:0:2355:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941503.905630:0:2355:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941503.905634:0:2355:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941503.905636:0:2355:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941503.905639:0:2355:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941503.905645:0:2355:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021ca80 x1860112241161600/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941503.905657:0:2355:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021ca80 x1860112241161600/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941503.905665:0:2355:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941503.905673:0:2355:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941503.905676:0:2355:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941503.905679:0:2355:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941503.905681:0:2355:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941503.905684:0:2355:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941503.905687:0:2355:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941503.905751:0:2355:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=82 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941503.905757:0:2355:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941503.905760:0:2355:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941503.905762:0:2355:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941503.905766:0:2355:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941503.905770:0:2355:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941503.905774:0:2355:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941503.905779:0:2355:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941503.905783:0:2355:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941503.905785:0:2355:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941503.905787:0:2355:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941503.905789:0:2355:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941503.905794:0:2355:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b9021ca80 x1860112241161600/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941503.905802:0:2355:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941503.905804:0:2355:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941503.905806:0:2355:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941503.905809:0:2355:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941503.905812:0:2355:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941503.905814:0:2355:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.905816:0:2355:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941503.905817:0:2355:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941503.905818:0:2355:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941503.905820:0:2355:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.905822:0:2355:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941503.905824:0:2355:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941503.905826:0:2355:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941503.905829:0:2355:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941503.905839:0:2355:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 273 00000400:00000200:3.0:1773941503.905844:0:2355:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941503.905849:0:2355:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941503.905853:0:2355:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941503.905856:0:2355:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941503.905859:0:2355:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941503.905862:0:2355:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941503.905865:0:2355:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941503.905867:0:2355:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941503.905870:0:2355:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941503.905873:0:2355:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941503.905876:0:2355:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941503.905879:0:2355:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941503.905882:0:2355:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941503.905884:0:2355:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941503.905886:0:2355:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941503.905889:0:2355:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941503.905892:0:2355:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941503.905895:0:2355:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941503.905898:0:2355:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941503.905901:0:2355:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941503.905904:0:2355:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941503.905907:0:2355:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.905909:0:2355:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941503.905911:0:2355:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941503.905916:0:2355:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90276300. 00000020:00000040:3.0:1773941503.905919:0:2355:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941503.905922:0:2355:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941503.905924:0:2355:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.905927:0:2355:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941503.905929:0:2355:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941503.905931:0:2355:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941503.905941:0:2355:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90262000. 00000100:00000001:3.0:1773941503.905945:0:2355:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941503.905951:0:2355:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90276300 x1860112241161728/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941503.905960:0:2355:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941503.905963:0:2355:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941503.905967:0:2355:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006af361f1] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90276300 x1860112241161728/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941503.905987:0:2355:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.905990:0:2355:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941503.905991:0:2355:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:3.0:1773941503.905997:0:2355:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021ca80 x1860112241161600/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941503.906006:0:2355:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941503.906009:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941503.906010:0:2355:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941503.906014:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941503.906015:0:2355:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941503.906018:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941503.906023:0:2355:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021ca80 x1860112241161600/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941519 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941503.906025:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90276300 x1860112241161728/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941503.906053:0:2355:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2355:1860112241161600:192.168.204.143@tcp:400:kworker.0 00000100:00000001:1.0:1773941503.906059:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941503.906062:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941503.906065:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941503.906066:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941503.906068:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.906071:0:2355:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941503.906072:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241161728:192.168.204.143@tcp:250:ptlrpcd_01_01.0 00000100:00000001:3.0:1773941503.906076:0:2355:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941503.906078:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941503.906081:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941503.906084:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941503.906085:0:2355:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021ca80 x1860112241161600/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941519 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941503.906086:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941503.906091:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0736800. 02000000:00000001:1.0:1773941503.906094:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941503.906102:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941503.906103:0:2355:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000400:00000010:1.0:1773941503.906105:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 02000000:00000001:3.0:1773941503.906107:0:2355:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000010:1.0:1773941503.906110:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 02000000:00000010:3.0:1773941503.906112:0:2355:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b839e2600. 02000000:00000001:3.0:1773941503.906116:0:2355:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941503.906119:0:2355:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941503.906123:0:2355:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000200:1.0:1773941503.906123:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241161728, portal 25 00000020:00000001:3.0:1773941503.906127:0:2355:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941503.906129:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941503.906131:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109626112 : -128249599925504 : ffff8b5b90276300) 02000000:00000010:3.0:1773941503.906138:0:2355:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b98bb0c00. 00000100:00000040:1.0:1773941503.906140:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90276300 x1860112241161728/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941503.906145:0:2355:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941503.906148:0:2355:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941503.906152:0:2355:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021ca80. 00000100:00000001:1.0:1773941503.906155:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1773941503.906157:0:2355:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941503.906162:0:2355:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941503.906162:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000100:00000001:3.0:1773941503.906166:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1773941503.906170:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1220. 00000100:00000001:3.0:1773941503.906173:0:2355:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000200:1.0:1773941503.906175:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241161728, offset 0 00000100:00000001:3.0:1773941503.906176:0:2355:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941503.906181:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000100:00000001:3.0:1773941503.906182:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941503.906187:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941503.906188:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941503.906193:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941503.906196:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941503.906199:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941503.906202:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941503.906206:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941503.906210:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[136, 137] 00000400:00000200:1.0:1773941503.906214:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941503.906217:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [243:243:256:4294967295] 192.168.204.143@tcp LPNI seq info [138:138:8:4294967295] 00000400:00000200:1.0:1773941503.906222:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941503.906230:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941503.906235:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941503.906240:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024bf00. 00000800:00000200:1.0:1773941503.906245:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941503.906252:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941503.906256:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024bf00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:1.0:1773941503.906267:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941503.906271:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941503.906272:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294944269 00000100:00000001:1.0:1773941503.906276:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294944267 00000100:00000001:1.0:1773941503.906288:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294944269 00000100:00000001:1.0:1773941503.906298:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294944267 00000100:00000001:1.0:1773941503.906302:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941503.906305:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.906308:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.906311:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941503.906315:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941503.906318:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1773941503.906320:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941503.906321:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941503.906322:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.906323:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941503.906342:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941503.906347:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024bf00. 00000400:00000200:2.0:1773941503.906351:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941503.906356:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941503.906359:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1220 00000400:00000010:2.0:1773941503.906361:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1220. 00000100:00000001:2.0:1773941503.906364:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941503.906368:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90276300 x1860112241161728/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941503.906376:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941503.906380:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90276300 x1860112241161728/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941503.906388:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941503.906390:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941503.907596:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.907602:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941503.907605:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.907607:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941503.907611:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941503.907615:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941503.907621:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941503.907624:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941503.907627:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941503.907631:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607600 00000400:00000200:2.0:1773941503.907635:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941503.907639:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x119d [1] + 0 00000800:00000001:2.0:1773941503.907650:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.907655:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941503.907657:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941503.907661:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941503.907665:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941503.907669:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90276300 x1860112241161728/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941503.907681:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90276300 x1860112241161728/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941503.907740:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941503.907744:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941503.907829:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941503.907843:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941503.907847:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:1.0:1773941503.907856:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941503.907859:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941503.907862:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941503.907867:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941503.907869:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941503.907872:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941503.907878:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90276300 x1860112241161728/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941503.907890:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90276300 x1860112241161728/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941503.907899:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941503.907900:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941503.907902:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941503.907904:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941503.907906:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941503.907907:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941503.907910:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941503.907913:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=82 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941503.907917:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941503.907919:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941503.907921:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941503.907930:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941503.907932:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.907935:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941503.907939:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90276300 x1860112241161728/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941503.907948:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941503.907951:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941503.907952:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941503.907955:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1773941503.907961:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:1.0:1773941503.907963:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941503.907965:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941503.907967:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941503.907969:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941503.907971:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941503.907974:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:1.0:1773941503.907979:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941503 v=0 (0 0 0 0) 00000100:00001000:1.0:1773941503.907982:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:1.0:1773941503.907988:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:1.0:1773941503.907991:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941503.907992:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:1.0:1773941503.907994:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.907996:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941503.908000:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:1.0:1773941503.908002:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:1.0:1773941503.908005:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:1.0:1773941503.908008:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:1.0:1773941503.908011:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:1.0:1773941503.908014:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941503.908354:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941503.908360:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941503.908368:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90276300 x1860112241161728/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941513 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941503.908378:0:12227:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:0.0:1773941503.908380:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241161728:192.168.204.143@tcp:250:ptlrpcd_01_01.0 00000100:00000001:0.0:1773941503.908388:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941503.908389:0:12227:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:0.0:1773941503.908393:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1773941503.908395:0:12227:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:0.0:1773941503.908400:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90276300 x1860112241161728/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941513 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941503.908405:0:12227:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941503.908409:0:12227:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941503.908414:0:12227:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:0.0:1773941503.908416:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:1.0:1773941503.908419:0:12227:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:0.0:1773941503.908420:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941503.908425:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0736800. 00000001:00000001:1.0:1773941503.908426:0:12227:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941503.908430:0:12227:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:0.0:1773941503.908430:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941503.908434:0:12227:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:0.0:1773941503.908434:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941503.908438:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:1.0:1773941503.908439:0:12227:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941503.908444:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000001:1.0:1773941503.908445:0:12227:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.908450:0:12227:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:0.0:1773941503.908450:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90262000. 00000100:00080000:1.0:1773941503.908455:0:12227:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:0.0:1773941503.908456:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941503.908460:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941503.908461:0:12227:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000010:0.0:1773941503.908464:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90276300. 00000100:00000001:1.0:1773941503.908465:0:12227:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941503.908469:0:12227:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941503.908471:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:1.0:1773941503.908473:0:12227:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941503.908475:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941503.908477:0:12227:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:0.0:1773941503.908479:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:1.0:1773941503.908481:0:12227:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941503.908483:0:12227:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:0.0:1773941503.908492:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941503.908494:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941503.908497:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1773941503.908499:0:12227:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:0.0:1773941503.908500:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941503.908503:0:12227:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.908506:0:12227:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941503.908511:0:12227:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941503.908516:0:12227:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941503.908519:0:12227:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941503.908521:0:12227:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941503.908524:0:12227:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941503.908526:0:12227:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941504.863952:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941504.863960:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941504.863965:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 00010000:00010000:3.0:1773941504.863966:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000040:1.0:1773941504.863972:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941504.863977:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941504.863980:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941504.863982:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941504.863983:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941504.863987:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941504.863989:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941504.863992:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941504.863995:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941504.864014:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941504.864021:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941504.864025:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941504.864030:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941504.864034:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941504.864039:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941504.864046:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941504.864052:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941504.864057:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941504.864060:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941504.864067:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890eca00. 00010000:00000001:1.0:1773941504.864072:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941504.864076:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941504.864080:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941504.864083:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1dcc to hash 00000020:00000001:1.0:1773941504.864085:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941504.864087:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941504.864090:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941504.864094:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941504.864097:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941504.864102:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dcc lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941504.864113:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dcc lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941504.864126:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dcc lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941504.864132:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941504.864134:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941504.864140:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941504.864142:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941504.864145:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941504.864150:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000001:1.0:1773941504.864155:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941504.864160:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941504.864163:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941504.864165:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941504.864174:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241161856/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941504.864185:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941504.864187:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941504.864191:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941504.864193:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941504.864196:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941504.864200:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241161856:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941504.864205:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941504.864210:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941504.864213:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941504.864216:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941504.864220:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0737000. 02000000:00000001:1.0:1773941504.864222:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941504.864229:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941504.864233:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941504.864240:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941504.864247:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241161856, portal 25 00000100:00000001:1.0:1773941504.864249:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941504.864250:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:1.0:1773941504.864255:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241161856/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941520 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941504.864265:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941504.864268:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941504.864270:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1f68. 00000100:00000200:1.0:1773941504.864273:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241161856, offset 0 00000400:00000200:1.0:1773941504.864280:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941504.864287:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941504.864298:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941504.864304:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941504.864308:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941504.864312:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941504.864315:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941504.864318:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[138, 137] 00000400:00000200:1.0:1773941504.864323:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941504.864327:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [244:244:256:4294967295] 192.168.204.153@tcp LPNI seq info [139:139:8:4294967295] 00000400:00000200:1.0:1773941504.864332:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941504.864343:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941504.864350:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941504.864355:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b400. 00000800:00000200:1.0:1773941504.864362:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941504.864370:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941504.864375:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b400 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941504.864390:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941504.864394:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941504.864396:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294945227 00000100:00000001:1.0:1773941504.864402:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294944269 00000100:00000001:1.0:1773941504.864406:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294945227 00000100:00000001:1.0:1773941504.864411:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294944269 00000100:00000001:1.0:1773941504.864415:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941504.864418:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941504.864421:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941504.864423:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941504.864426:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941504.864430:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941504.864432:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941504.864519:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941504.864525:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b400. 00000400:00000200:2.0:1773941504.864531:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941504.864538:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941504.864543:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1f68 00000400:00000010:2.0:1773941504.864546:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1f68. 00000100:00000001:2.0:1773941504.864551:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941504.864556:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241161856/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941520 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941504.864567:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941504.864571:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d63800 x1860112241161856/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941520 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941504.864579:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941504.864582:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941504.865635:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941504.865647:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941504.865650:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941504.865652:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941504.865658:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941504.865664:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941504.865671:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941504.865699:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941504.865722:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941504.865727:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607680 00000400:00000200:2.0:1773941504.865732:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941504.865738:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x11a5 [1] + 224 00000800:00000001:2.0:1773941504.865744:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941504.865753:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941504.865756:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941504.865760:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941504.865765:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941504.865769:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241161856/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941520 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941504.865781:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d63800 x1860112241161856/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941520 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941504.865799:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941504.865806:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941504.865810:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941504.865834:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941504.865840:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941504.865842:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941504.865851:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941504.865855:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941504.865857:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941504.865860:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941504.865864:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241161856/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941520 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941504.865875:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241161856/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941520 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941504.865883:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941504.865884:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941504.865887:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941504.865890:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941504.865892:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941504.865895:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941504.865898:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941504.865903:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=83 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941504.865906:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941504.865910:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941504.865914:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941504.865916:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941504.865920:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941504.865923:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941504.865927:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941504.865932:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941504.865936:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941504.865938:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941504.865940:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941504.865942:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941504.865946:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d63800 x1860112241161856/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941520 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941504.865953:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941504.865972:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941504.865974:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941504.865977:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941504.865980:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941504.865982:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941504.865984:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941504.865985:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941504.865986:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941504.865988:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941504.865989:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941504.865992:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941504.865995:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941504.865998:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941504.866001:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 274 00000400:00000200:1.0:1773941504.866005:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941504.866010:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941504.866014:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941504.866017:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941504.866020:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941504.866022:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941504.866025:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941504.866027:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941504.866029:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941504.866031:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941504.866033:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941504.866036:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941504.866039:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941504.866041:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941504.866043:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941504.866046:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941504.866049:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941504.866052:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941504.866055:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941504.866061:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941504.866064:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941504.866067:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941504.866069:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941504.866071:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941504.866075:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60e00. 00000020:00000040:1.0:1773941504.866077:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941504.866081:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941504.866083:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941504.866085:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941504.866087:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941504.866089:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941504.866093:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734000. 00000100:00000001:1.0:1773941504.866096:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941504.866102:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60e00 x1860112241161984/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941504.866111:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941504.866114:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941504.866118:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5e4d003] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60e00 x1860112241161984/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941504.866140:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941504.866143:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941504.866145:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941504.866147:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941504.866150:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941504.866151:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941504.866153:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941504.866155:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941504.866157:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941504.866162:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241161856/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941520 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941504.866171:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941504.866175:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941504.866181:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941504.866183:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941504.866189:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241161856/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941520 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941504.866191:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941504.866196:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941504.866200:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:1.0:1773941504.866204:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241161856:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000040:0.0:1773941504.866211:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60e00 x1860112241161984/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941504.866212:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941504.866217:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941504.866223:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941504.866226:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941504.866230:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1773941504.866235:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241161856/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941520 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941504.866235:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941504.866241:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941504.866245:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941504.866250:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941504.866251:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941504.866254:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941504.866259:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00100000:0.0:1773941504.866260:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241161984:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941504.866261:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941504.866267:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:0.0:1773941504.866268:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941504.866271:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:0.0:1773941504.866273:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1773941504.866274:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:0.0:1773941504.866276:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941504.866278:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 02000000:00000001:0.0:1773941504.866280:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:1.0:1773941504.866284:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941504.866288:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 02000000:00000010:0.0:1773941504.866288:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0c00. 02000000:00000001:0.0:1773941504.866293:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941504.866296:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dcc lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941504.866301:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941504.866308:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00000001:1.0:1773941504.866311:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000010:0.0:1773941504.866315:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00010000:00010000:1.0:1773941504.866317:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dcc lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:0.0:1773941504.866321:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241161984, portal 25 00000100:00000001:0.0:1773941504.866327:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941504.866331:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:0.0:1773941504.866331:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00010000:00010000:1.0:1773941504.866336:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dcc lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941504.866341:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60e00 x1860112241161984/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941514 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941504.866364:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941504.866364:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941504.866368:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000040:0.0:1773941504.866370:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941504.866373:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dcc lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941504.866375:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6cbb0. 00000100:00000200:0.0:1773941504.866382:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241161984, offset 0 00010000:00000001:1.0:1773941504.866389:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941504.866393:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941504.866394:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941504.866396:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941504.866400:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dcc lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941504.866411:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941504.866412:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941504.866417:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941504.866420:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 10000000:00010000:1.0:1773941504.866422:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dcc lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941504.866425:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941504.866431:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941504.866436:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941504.866437:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941504.866441:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:0.0:1773941504.866442:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1773941504.866444:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941504.866449:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000400:00000200:0.0:1773941504.866450:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[138, 139] 00000020:00000001:1.0:1773941504.866454:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000400:00000200:0.0:1773941504.866458:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941504.866460:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941504.866465:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:0.0:1773941504.866469:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [245:245:256:4294967295] 192.168.204.143@tcp LPNI seq info [140:140:8:4294967295] 00010000:00010000:1.0:1773941504.866470:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dcc lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941504.866481:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:1.0:1773941504.866487:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941504.866492:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dcc lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941504.866496:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 10000000:01000000:1.0:1773941504.866508:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000800:00000200:0.0:1773941504.866508:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941504.866514:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 10000000:00000040:1.0:1773941504.866518:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941504.866521:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941504.866523:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00010000:1.0:1773941504.866525:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941504.866529:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941504.866532:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000800:00000200:0.0:1773941504.866535:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941504.866536:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941504.866540:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000800:00000200:0.0:1773941504.866542:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941504.866543:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941504.866551:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1dcc from hash 00010000:00000001:1.0:1773941504.866554:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941504.866556:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941504.866557:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941504.866558:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941504.866560:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941504.866563:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:0.0:1773941504.866563:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941504.866569:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941504.866572:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:0.0:1773941504.866572:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941504.866575:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941504.866577:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294945229 10000000:00000001:1.0:1773941504.866580:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941504.866584:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294945227 00010000:00010000:1.0:1773941504.866585:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dcc lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941504.866592:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294945229 00010000:00000001:1.0:1773941504.866595:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941504.866598:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941504.866599:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294945227 00010000:00000001:1.0:1773941504.866600:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941504.866603:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941504.866606:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941504.866608:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941504.866608:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941504.866611:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941504.866614:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941504.866616:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dcc lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941504.866619:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941504.866626:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:1.0:1773941504.866628:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890eca00 count: 0 00000800:00000200:2.0:1773941504.866635:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1773941504.866636:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000800:00000010:2.0:1773941504.866640:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000100:00000001:0.0:1773941504.866641:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:1.0:1773941504.866642:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941504.866646:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941504.866647:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1773941504.866648:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941504.866652:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941504.866652:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1773941504.866656:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:0.0:1773941504.866657:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941504.866662:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6cbb0 00000400:00000010:2.0:1773941504.866665:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6cbb0. 00000100:00000040:1.0:1773941504.866665:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241161856/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941520 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941504.866668:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941504.866671:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941504.866696:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60e00 x1860112241161984/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941514 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941504.866699:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941504.866724:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941504.866728:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60e00 x1860112241161984/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941514 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941504.866734:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941504.866736:0:2347:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1773941504.866742:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941504.866746:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0737000. 02000000:00000001:1.0:1773941504.866750:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941504.866752:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941504.866754:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941504.866757:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941504.866760:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941504.866764:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941504.866765:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941504.866767:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:1.0:1773941504.866771:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941504.866772:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941504.866775:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941504.866778:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941504.866781:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941504.866786:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0500. 00000020:00000010:1.0:1773941504.866790:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941504.866795:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941504.866800:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941504.866805:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941504.866810:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941504.866814:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941504.866819:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941504.866825:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941504.866830:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941504.866832:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941504.866836:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941504.866840:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941504.866843:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941504.866847:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941504.866850:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941504.866860:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941504.866863:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941504.866866:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941504.866869:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0500. 10000000:01000000:1.0:1773941504.866872:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941504.866875:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941504.866877:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941504.866879:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941504.866881:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941504.866882:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941504.866885:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941504.867986:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941504.867992:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941504.867994:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941504.867996:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941504.867999:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941504.868005:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941504.868011:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941504.868014:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941504.868017:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941504.868021:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607700 00000400:00000200:2.0:1773941504.868025:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941504.868030:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x11ad [1] + 0 00000800:00000001:2.0:1773941504.868035:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941504.868042:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941504.868044:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941504.868048:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941504.868051:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941504.868054:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60e00 x1860112241161984/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941514 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941504.868067:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60e00 x1860112241161984/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941504.868078:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941504.868082:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941504.868084:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941504.868120:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941504.868123:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941504.868126:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941504.868128:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941504.868130:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941504.868134:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941504.868136:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941504.868139:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941504.868144:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60e00 x1860112241161984/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941504.868156:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60e00 x1860112241161984/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941504.868163:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941504.868165:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941504.868167:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941504.868169:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941504.868170:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941504.868172:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941504.868174:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941504.868178:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=83 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941504.868181:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941504.868184:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941504.868186:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941504.868188:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941504.868190:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941504.868194:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941504.868199:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60e00 x1860112241161984/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941504.868206:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941504.868208:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941504.868210:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941504.868212:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941504.868216:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941504.868219:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941504.868226:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941504.868228:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941504.868230:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941504.868232:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941504.868235:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941504.868239:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941504 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941504.868242:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941504.868248:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941504.868250:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941504.868252:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941504.868254:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941504.868256:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941504.868260:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941504.868262:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941504.868265:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941504.868268:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941504.868272:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941504.868274:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941504.868514:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941504.868518:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941504.868524:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60e00 x1860112241161984/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941514 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941504.868535:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241161984:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941504.868541:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941504.868543:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941504.868544:0:12228:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:1.0:1773941504.868549:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60e00 x1860112241161984/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941514 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941504.868557:0:12228:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:1.0:1773941504.868559:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:0.0:1773941504.868563:0:12228:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:1.0:1773941504.868564:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941504.868567:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0c00. 00000100:00000001:0.0:1773941504.868568:0:12228:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:1.0:1773941504.868572:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:01000000:0.0:1773941504.868573:0:12228:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000001:1.0:1773941504.868576:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941504.868578:0:12228:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000040:1.0:1773941504.868579:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941504.868582:0:12228:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:1.0:1773941504.868583:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941504.868587:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734000. 00000001:00000001:0.0:1773941504.868588:0:12228:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941504.868591:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:0.0:1773941504.868592:0:12228:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:1.0:1773941504.868594:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:0.0:1773941504.868595:0:12228:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000010:1.0:1773941504.868598:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60e00. 00000001:00000001:0.0:1773941504.868600:0:12228:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941504.868604:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 10000000:00000001:0.0:1773941504.868604:0:12228:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941504.868608:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941504.868609:0:12228:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941504.868614:0:12228:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941504.868618:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941504.868619:0:12228:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941504.868624:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941504.868626:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941504.868631:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941504.868632:0:12228:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941504.868634:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941504.868637:0:12228:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941504.868641:0:12228:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941504.868644:0:12228:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941504.868646:0:12228:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941504.868647:0:12228:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941504.868661:0:12228:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941504.868663:0:12228:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941504.868665:0:12228:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941504.868669:0:12228:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941504.868687:0:12228:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941504.868690:0:12228:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941504.868692:0:12228:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941504.868695:0:12228:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941504.868697:0:12228:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941504.871738:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890eca00. 00010000:00000010:1.1:1773941504.871745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:0.0:1773941505.887894:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941505.887904:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941505.887911:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941506.015841:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941506.015852:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941506.015858:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941506.015862:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941506.015864:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941506.015866:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941506.015869:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941506.015872:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941506.015875:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941506.015878:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941506.015881:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941506.015898:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60e00. 00000020:00000040:1.0:1773941506.015905:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941506.015911:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941506.015916:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941506.015924:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941506.015934:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941506.015953:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941506.015959:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941506.015963:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941506.015974:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890eca00. 00010000:00000001:1.0:1773941506.015982:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941506.015987:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941506.015992:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941506.015995:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1dd3 to hash 00000020:00000001:1.0:1773941506.015998:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941506.016000:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941506.016004:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941506.016010:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941506.016014:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941506.016019:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dd3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941506.016053:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dd3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941506.016067:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dd3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941506.016075:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941506.016077:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941506.016086:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941506.016088:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941506.016092:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941506.016094:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00000100:00000001:1.0:1773941506.016100:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941506.016105:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941506.016108:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941506.016110:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941506.016125:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60e00 x1860112241162112/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941506.016135:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941506.016138:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941506.016143:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941506.016146:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941506.016149:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941506.016154:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241162112:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941506.016160:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941506.016167:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941506.016170:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941506.016173:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941506.016178:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734000. 02000000:00000001:1.0:1773941506.016181:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941506.016191:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941506.016202:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941506.016212:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941506.016222:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241162112, portal 25 00000100:00000001:1.0:1773941506.016225:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941506.016226:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00000100:00000040:1.0:1773941506.016234:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60e00 x1860112241162112/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941522 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941506.016243:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941506.016246:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941506.016250:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1550. 00000100:00000200:1.0:1773941506.016254:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241162112, offset 0 00000400:00000200:1.0:1773941506.016262:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941506.016273:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941506.016283:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941506.016290:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941506.016295:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941506.016300:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941506.016304:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941506.016307:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[140, 139] 00000400:00000200:1.0:1773941506.016312:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941506.016317:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [246:246:256:4294967295] 192.168.204.153@tcp LPNI seq info [141:141:8:4294967295] 00000400:00000200:1.0:1773941506.016322:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941506.016335:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941506.016342:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941506.016349:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b600. 00000800:00000200:1.0:1773941506.016358:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941506.016367:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941506.016372:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941506.016396:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941506.016401:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941506.016404:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294946379 00000100:00000001:1.0:1773941506.016420:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294945229 00000100:00000001:1.0:1773941506.016424:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294946379 00000100:00000001:1.0:1773941506.016428:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294945229 00000100:00000001:1.0:1773941506.016432:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941506.016435:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941506.016437:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941506.016439:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941506.016443:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941506.016446:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941506.016450:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941506.016577:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941506.016582:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b600. 00000400:00000200:2.0:1773941506.016588:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941506.016594:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941506.016599:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1550 00000400:00000010:2.0:1773941506.016601:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1550. 00000100:00000001:2.0:1773941506.016606:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941506.016611:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60e00 x1860112241162112/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941522 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941506.016630:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941506.016633:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d60e00 x1860112241162112/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941522 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941506.016639:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941506.016641:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941506.018062:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941506.018078:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941506.018081:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941506.018085:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941506.018091:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941506.018096:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941506.018105:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941506.018110:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941506.018115:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941506.018119:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607780 00000400:00000200:2.0:1773941506.018126:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941506.018133:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x11b5 [1] + 224 00000800:00000001:2.0:1773941506.018142:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941506.018152:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941506.018155:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941506.018159:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941506.018164:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941506.018169:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60e00 x1860112241162112/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941522 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941506.018180:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d60e00 x1860112241162112/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941522 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941506.018197:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941506.018205:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941506.018209:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941506.018223:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941506.018229:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941506.018231:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941506.018233:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941506.018236:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941506.018238:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941506.018239:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941506.018244:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60e00 x1860112241162112/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941522 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941506.018252:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60e00 x1860112241162112/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941522 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941506.018258:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941506.018259:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941506.018261:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941506.018264:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941506.018266:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941506.018268:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941506.018271:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941506.018276:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=85 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941506.018279:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941506.018282:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941506.018285:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941506.018288:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941506.018296:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941506.018298:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941506.018302:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941506.018306:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941506.018309:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941506.018311:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941506.018312:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941506.018314:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941506.018317:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d60e00 x1860112241162112/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941522 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941506.018322:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941506.018323:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941506.018325:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941506.018327:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941506.018330:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941506.018331:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941506.018334:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941506.018335:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941506.018336:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941506.018337:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941506.018338:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941506.018341:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941506.018344:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941506.018347:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941506.018350:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 275 00000400:00000200:1.0:1773941506.018355:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941506.018360:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941506.018365:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941506.018369:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941506.018372:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941506.018374:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941506.018378:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941506.018385:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941506.018388:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941506.018391:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941506.018394:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941506.018396:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941506.018400:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941506.018402:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941506.018404:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941506.018406:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941506.018409:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941506.018413:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941506.018417:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941506.018420:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941506.018423:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941506.018426:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941506.018429:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941506.018431:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941506.018436:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941506.018439:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941506.018442:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941506.018444:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941506.018446:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941506.018447:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941506.018449:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941506.018452:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737000. 00000100:00000001:1.0:1773941506.018455:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941506.018461:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63800 x1860112241162240/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941506.018467:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941506.018469:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941506.018473:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc928f9c] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63800 x1860112241162240/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941506.018501:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941506.018503:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941506.018504:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941506.018507:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941506.018509:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941506.018510:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941506.018512:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941506.018514:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941506.018516:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941506.018521:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60e00 x1860112241162112/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941522 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941506.018527:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941506.018527:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941506.018530:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941506.018533:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941506.018533:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941506.018537:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941506.018538:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941506.018547:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60e00 x1860112241162112/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941522 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941506.018548:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241162240/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941506.018566:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241162112:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941506.018573:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941506.018579:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941506.018579:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941506.018583:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941506.018584:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941506.018587:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1773941506.018588:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1773941506.018592:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60e00 x1860112241162112/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941522 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1773941506.018593:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941506.018598:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941506.018606:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241162240:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941506.018607:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941506.018613:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:0.0:1773941506.018614:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941506.018618:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:0.0:1773941506.018618:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941506.018621:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:0.0:1773941506.018622:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941506.018628:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941506.018629:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000010:0.0:1773941506.018633:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c3000. 00010000:00000001:1.0:1773941506.018637:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:0.0:1773941506.018637:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941506.018640:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941506.018642:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 00000020:00000001:1.0:1773941506.018643:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941506.018649:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00010000:1.0:1773941506.018667:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dd3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941506.018669:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941506.018676:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00000001:1.0:1773941506.018678:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000010:0.0:1773941506.018683:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00010000:00010000:1.0:1773941506.018684:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dd3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:0.0:1773941506.018691:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241162240, portal 25 00010000:00000001:1.0:1773941506.018692:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941506.018696:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dd3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941506.018697:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941506.018701:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:0.0:1773941506.018733:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241162240/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941516 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941506.018734:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941506.018737:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941506.018740:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dd3 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941506.018753:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941506.018756:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941506.018758:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941506.018761:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000040:0.0:1773941506.018762:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941506.018765:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dd3 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941506.018767:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6cd48. 00000100:00000200:0.0:1773941506.018773:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241162240, offset 0 00010000:00010000:1.0:1773941506.018775:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941506.018779:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941506.018780:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 10000000:00010000:1.0:1773941506.018782:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dd3 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941506.018789:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941506.018796:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941506.018804:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941506.018806:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941506.018810:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:0.0:1773941506.018810:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000020:00000001:1.0:1773941506.018812:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941506.018816:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000400:00000200:0.0:1773941506.018819:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1773941506.018820:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941506.018825:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000400:00000200:0.0:1773941506.018828:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941506.018829:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941506.018834:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dd3 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941506.018836:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[140, 141] 00000400:00000200:0.0:1773941506.018849:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 10000000:00000001:1.0:1773941506.018852:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941506.018857:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dd3 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941506.018857:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [247:247:256:4294967295] 192.168.204.143@tcp LPNI seq info [142:142:8:4294967295] 00000400:00000200:0.0:1773941506.018866:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:01000000:1.0:1773941506.018874:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:0.0:1773941506.018876:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 10000000:00000040:1.0:1773941506.018883:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000800:00000200:0.0:1773941506.018885:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 10000000:00000001:1.0:1773941506.018888:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1773941506.018891:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00010000:00010000:1.0:1773941506.018894:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941506.018898:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000800:00000200:0.0:1773941506.018900:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941506.018903:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941506.018907:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941506.018910:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000800:00000200:0.0:1773941506.018911:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941506.018913:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941506.018918:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1dd3 from hash 00000800:00000200:0.0:1773941506.018918:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941506.018923:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941506.018927:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941506.018928:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941506.018929:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941506.018931:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941506.018932:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941506.018934:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941506.018935:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941506.018936:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941506.018939:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941506.018945:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941506.018950:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941506.018951:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dd3 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941506.018957:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941506.018959:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941506.018963:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:0.0:1773941506.018964:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294946382 00010000:00000001:1.0:1773941506.018965:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941506.018968:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941506.018971:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941506.018972:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294946379 00010000:00000001:1.0:1773941506.018973:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941506.018977:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dd3 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941506.018980:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294946382 00010000:00000040:1.0:1773941506.018988:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890eca00 count: 0 00000100:00000001:0.0:1773941506.018998:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294946379 00000100:00000001:0.0:1773941506.019003:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941506.019006:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941506.019007:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941506.019010:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941506.019011:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941506.019011:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941506.019016:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941506.019017:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000800:00000010:2.0:1773941506.019024:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000100:00000040:1.0:1773941506.019028:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60e00 x1860112241162112/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941522 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941506.019029:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:2.0:1773941506.019031:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1773941506.019033:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1773941506.019040:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1773941506.019045:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:0.0:1773941506.019045:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941506.019049:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941506.019052:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941506.019054:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941506.019055:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6cd48 02000000:00000001:1.0:1773941506.019057:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000010:2.0:1773941506.019062:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6cd48. 02000000:00000010:1.0:1773941506.019063:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734000. 00000100:00000001:2.0:1773941506.019069:0:2347:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1773941506.019070:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941506.019075:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941506.019079:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000200:2.0:1773941506.019080:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241162240/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941516 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1773941506.019086:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941506.019092:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 00000100:00000001:2.0:1773941506.019093:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941506.019098:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241162240/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941516 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941506.019099:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941506.019103:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941506.019107:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60e00. 00000100:00000001:2.0:1773941506.019108:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941506.019112:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1773941506.019114:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941506.019117:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941506.019120:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941506.019124:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941506.019128:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941506.019141:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0500. 00000020:00000010:1.0:1773941506.019146:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941506.019152:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941506.019157:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941506.019163:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941506.019169:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941506.019175:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941506.019181:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941506.019186:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 10000000:00000040:1.0:1773941506.019192:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941506.019195:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941506.019200:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941506.019205:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941506.019208:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941506.019213:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941506.019216:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941506.019222:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941506.019225:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941506.019228:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941506.019232:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0500. 10000000:01000000:1.0:1773941506.019236:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941506.019239:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941506.019242:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941506.019244:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941506.019246:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941506.019248:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941506.019250:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941506.020293:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941506.020308:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941506.020313:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941506.020315:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941506.020322:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941506.020337:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941506.020346:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941506.020351:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941506.020357:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941506.020360:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607800 00000400:00000200:2.0:1773941506.020364:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941506.020368:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x11bd [1] + 0 00000800:00000001:2.0:1773941506.020375:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941506.020382:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941506.020385:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941506.020388:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941506.020394:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941506.020401:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241162240/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941516 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941506.020410:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63800 x1860112241162240/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941516 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941506.020438:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941506.020446:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941506.020448:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941506.020476:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941506.020478:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941506.020482:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941506.020483:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941506.020485:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941506.020488:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941506.020490:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941506.020492:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941506.020498:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241162240/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941516 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941506.020506:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241162240/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941516 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941506.020512:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941506.020513:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941506.020514:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941506.020516:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941506.020517:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941506.020518:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941506.020520:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941506.020522:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=85 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941506.020525:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941506.020532:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941506.020534:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941506.020536:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941506.020538:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941506.020540:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941506.020543:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241162240/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941516 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941506.020549:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941506.020551:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941506.020552:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941506.020554:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941506.020557:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941506.020559:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941506.020560:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941506.020561:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941506.020563:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941506.020564:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941506.020566:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941506.020569:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941506 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941506.020571:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941506.020577:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941506.020579:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941506.020580:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941506.020582:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941506.020583:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941506.020586:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941506.020588:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941506.020589:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941506.020592:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941506.020594:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941506.020596:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941506.020866:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941506.020871:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941506.020878:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241162240/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941516 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941506.020889:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241162240:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941506.020892:0:12229:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:1.0:1773941506.020906:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941506.020910:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1773941506.020913:0:12229:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000040:1.0:1773941506.020918:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241162240/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941516 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941506.020920:0:12229:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941506.020926:0:12229:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941506.020929:0:12229:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:1.0:1773941506.020932:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941506.020935:0:12229:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:1.0:1773941506.020936:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941506.020940:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c3000. 00000001:00000001:0.0:1773941506.020941:0:12229:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:1.0:1773941506.020944:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941506.020948:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941506.020948:0:12229:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941506.020952:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941506.020954:0:12229:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:1.0:1773941506.020956:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:0.0:1773941506.020958:0:12229:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:1.0:1773941506.020961:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737000. 00000001:00000001:0.0:1773941506.020962:0:12229:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941506.020966:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000001:0.0:1773941506.020968:0:12229:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941506.020971:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941506.020973:0:12229:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000010:1.0:1773941506.020974:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:1.0:1773941506.020980:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:0.0:1773941506.020980:0:12229:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941506.020984:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941506.020987:0:12229:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941506.020989:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941506.020992:0:12229:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941506.020995:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941506.020997:0:12229:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941506.020999:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941506.021002:0:12229:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941506.021004:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941506.021007:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941506.021010:0:12229:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941506.021013:0:12229:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941506.021016:0:12229:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941506.021034:0:12229:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941506.021035:0:12229:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941506.021037:0:12229:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941506.021041:0:12229:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941506.021043:0:12229:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941506.021047:0:12229:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941506.021048:0:12229:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941506.021050:0:12229:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941506.021053:0:12229:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941506.024745:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890eca00. 00010000:00000010:1.1:1773941506.024755:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:3.0:1773941506.911861:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941506.911867:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941506.911871:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1773941507.935881:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941507.935888:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941507.935893:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941507.999818:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941507.999825:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941507.999829:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941507.999832:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941507.999833:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941507.999834:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941507.999836:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941507.999838:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941507.999840:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941507.999842:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941507.999843:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941507.999853:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941507.999857:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941507.999861:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941507.999879:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941507.999885:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941507.999891:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941507.999896:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941507.999900:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941507.999902:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941507.999908:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890eca00. 00010000:00000001:1.0:1773941507.999913:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941507.999916:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:1.0:1773941507.999919:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941507.999921:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138f600 with handle 0xa33a5be14eae1dda to hash 00000020:00000001:1.0:1773941507.999922:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941507.999923:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941507.999926:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00010000:00000001:1.0:1773941507.999929:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941507.999932:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941507.999935:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dda lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941507.999942:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dda lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941507.999952:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dda lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941507.999956:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941507.999957:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941507.999962:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941507.999963:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941507.999965:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941507.999966:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000001:1.0:1773941507.999970:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941507.999974:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941507.999975:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941507.999980:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941507.999989:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241162368/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941507.999996:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941507.999998:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941508.000001:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941508.000003:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941508.000005:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941508.000007:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241162368:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941508.000010:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941508.000014:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941508.000016:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941508.000018:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941508.000021:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734000. 02000000:00000001:1.0:1773941508.000022:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941508.000028:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941508.000031:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941508.000037:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941508.000042:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241162368, portal 25 00000100:00000001:1.0:1773941508.000044:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941508.000045:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:1.0:1773941508.000049:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241162368/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941523 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941508.000054:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941508.000056:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941508.000058:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1d48. 00000100:00000200:1.0:1773941508.000060:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241162368, offset 0 00000400:00000200:1.0:1773941508.000065:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941508.000072:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941508.000078:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941508.000082:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941508.000086:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941508.000089:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941508.000094:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941508.000097:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[142, 141] 00000400:00000200:1.0:1773941508.000100:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941508.000103:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [248:248:256:4294967295] 192.168.204.153@tcp LPNI seq info [143:143:8:4294967295] 00000400:00000200:1.0:1773941508.000106:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941508.000115:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941508.000119:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941508.000122:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024bb00. 00000800:00000200:1.0:1773941508.000128:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941508.000134:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941508.000138:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024bb00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941508.000154:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941508.000157:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941508.000159:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294948363 00000100:00000001:1.0:1773941508.000165:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294946382 00000100:00000001:1.0:1773941508.000168:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294948363 00000100:00000001:1.0:1773941508.000171:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294946382 00000100:00000001:1.0:1773941508.000173:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941508.000175:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941508.000177:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941508.000178:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941508.000180:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941508.000183:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941508.000186:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941508.000272:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941508.000277:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024bb00. 00000400:00000200:2.0:1773941508.000282:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941508.000286:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941508.000291:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1d48 00000400:00000010:2.0:1773941508.000293:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1d48. 00000100:00000001:2.0:1773941508.000296:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941508.000301:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241162368/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941523 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941508.000309:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941508.000311:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d63800 x1860112241162368/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941523 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941508.000316:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941508.000318:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941508.001596:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941508.001606:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941508.001609:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941508.001610:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941508.001625:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941508.001630:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941508.001635:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941508.001638:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941508.001641:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941508.001644:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607880 00000400:00000200:2.0:1773941508.001647:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941508.001651:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x11c5 [1] + 224 00000800:00000001:2.0:1773941508.001656:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941508.001671:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941508.001673:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941508.001681:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941508.001684:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941508.001688:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241162368/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941523 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941508.001696:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d63800 x1860112241162368/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941523 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941508.001732:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941508.001739:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941508.001742:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941508.001744:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941508.001750:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941508.001752:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941508.001755:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941508.001759:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941508.001760:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941508.001762:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941508.001773:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241162368/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941523 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941508.001781:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241162368/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941523 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941508.001786:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941508.001788:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941508.001790:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941508.001791:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941508.001794:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941508.001795:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941508.001797:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941508.001801:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=87 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941508.001804:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941508.001806:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941508.001809:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941508.001811:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941508.001813:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941508.001815:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941508.001819:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941508.001822:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941508.001824:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941508.001826:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941508.001827:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941508.001829:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941508.001831:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d63800 x1860112241162368/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941523 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941508.001836:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941508.001837:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941508.001838:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941508.001840:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941508.001842:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941508.001843:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941508.001848:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941508.001849:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941508.001849:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941508.001850:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941508.001851:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941508.001854:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941508.001855:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941508.001858:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941508.001860:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 276 00000400:00000200:1.0:1773941508.001863:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941508.001866:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941508.001869:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941508.001871:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941508.001873:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941508.001874:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941508.001876:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941508.001878:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941508.001879:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941508.001881:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941508.001883:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941508.001884:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941508.001886:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941508.001887:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941508.001888:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941508.001889:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941508.001891:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941508.001893:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941508.001895:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941508.001897:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941508.001899:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941508.001901:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941508.001902:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941508.001908:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941508.001910:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60e00. 00000020:00000040:1.0:1773941508.001913:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941508.001915:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941508.001916:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941508.001918:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941508.001919:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941508.001921:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941508.001924:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734c00. 00000100:00000001:1.0:1773941508.001926:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941508.001931:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60e00 x1860112241162496/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941508.001937:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941508.001939:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941508.001942:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5e4d003] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60e00 x1860112241162496/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941508.001961:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941508.001963:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941508.001964:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941508.001966:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941508.001968:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941508.001969:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941508.001970:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941508.001971:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941508.001973:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941508.001976:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241162368/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941523 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941508.001981:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941508.001983:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941508.001984:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941508.001986:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941508.001990:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241162368/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941523 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941508.001999:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941508.002003:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941508.002006:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:1.0:1773941508.002009:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241162368:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000040:0.0:1773941508.002014:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60e00 x1860112241162496/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941508.002018:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941508.002021:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941508.002028:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941508.002031:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941508.002034:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941508.002036:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941508.002039:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1773941508.002040:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241162368/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941523 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1773941508.002041:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941508.002045:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941508.002052:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241162496:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941508.002061:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941508.002061:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941508.002063:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941508.002066:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941508.002067:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 02000000:00000001:0.0:1773941508.002068:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941508.002073:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0000. 00000100:00000001:1.0:1773941508.002074:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:0.0:1773941508.002076:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941508.002078:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:0.0:1773941508.002083:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941508.002086:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000010:0.0:1773941508.002086:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00000001:1.0:1773941508.002092:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000010:0.0:1773941508.002093:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000020:00000001:1.0:1773941508.002096:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:0.0:1773941508.002098:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241162496, portal 25 00000020:00000040:1.0:1773941508.002100:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=4 00000100:00000001:0.0:1773941508.002101:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941508.002103:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00000020:00000001:1.0:1773941508.002107:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000040:0.0:1773941508.002110:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60e00 x1860112241162496/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941518 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941508.002112:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000001:0.0:1773941508.002121:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:1.0:1773941508.002122:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dda lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941508.002124:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941508.002128:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c550. 00000100:00000200:0.0:1773941508.002132:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241162496, offset 0 00000400:00000200:0.0:1773941508.002139:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941508.002140:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:0.0:1773941508.002145:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941508.002150:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dda lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941508.002150:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941508.002156:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941508.002161:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941508.002164:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941508.002168:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:0.0:1773941508.002168:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:1.0:1773941508.002173:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dda lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941508.002178:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[142, 143] 00000400:00000200:0.0:1773941508.002188:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941508.002194:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941508.002195:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [249:249:256:4294967295] 192.168.204.143@tcp LPNI seq info [144:144:8:4294967295] 00000400:00000200:0.0:1773941508.002198:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941508.002205:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941508.002207:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:0.0:1773941508.002211:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941508.002213:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dda lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:0.0:1773941508.002215:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00000800:00000200:0.0:1773941508.002224:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941508.002230:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000800:00000200:0.0:1773941508.002234:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941508.002235:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941508.002239:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941508.002241:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941508.002246:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dda lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941508.002251:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941508.002257:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138f600 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:0.0:1773941508.002257:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941508.002261:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294948365 10000000:00000001:1.0:1773941508.002263:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:0.0:1773941508.002266:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294948363 10000000:00010000:1.0:1773941508.002268:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dda lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941508.002270:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294948365 00000100:00000001:0.0:1773941508.002275:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294948363 00000100:00000001:0.0:1773941508.002280:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941508.002283:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:0.0:1773941508.002283:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941508.002289:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:0.0:1773941508.002289:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941508.002294:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1773941508.002294:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1773941508.002299:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138f600 refcount=5 00000020:00000001:1.0:1773941508.002304:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000001:0.0:1773941508.002304:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941508.002306:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:1.0:1773941508.002309:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859119616 : -128249850432000 : ffff8b5b8138f600) 00000100:00000001:0.0:1773941508.002311:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941508.002313:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941508.002316:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000001:0.0:1773941508.002316:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941508.002321:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941508.002323:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dda lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:1.0:1773941508.002332:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941508.002335:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dda lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941508.002342:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941508.002346:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941508.002348:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941508.002350:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00010000:1.0:1773941508.002352:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000800:00000010:2.0:1773941508.002355:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00010000:00000001:1.0:1773941508.002356:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941508.002361:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941508.002366:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00010000:00000001:1.0:1773941508.002367:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:2.0:1773941508.002370:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c550 00010000:00000001:1.0:1773941508.002371:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000010:2.0:1773941508.002374:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c550. 00010000:00000001:1.0:1773941508.002375:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:2.0:1773941508.002378:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941508.002379:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941508.002383:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60e00 x1860112241162496/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941518 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1773941508.002385:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138f600 with handle 0xa33a5be14eae1dda from hash 00010000:00000001:1.0:1773941508.002391:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941508.002394:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941508.002396:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000001:1.0:1773941508.002398:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000040:2.0:1773941508.002400:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60e00 x1860112241162496/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941518 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941508.002401:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941508.002405:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941508.002409:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:2.0:1773941508.002411:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941508.002414:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941508.002415:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1773941508.002418:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941508.002421:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941508.002424:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941508.002428:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dda lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941508.002434:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941508.002436:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941508.002437:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941508.002438:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941508.002440:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941508.002441:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941508.002442:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941508.002444:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138f600/0xa33a5be14eae1dda lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941508.002451:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890eca00 count: 0 00010000:00000001:1.0:1773941508.002463:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941508.002466:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941508.002468:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941508.002474:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241162368/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941523 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941508.002483:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941508.002486:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941508.002496:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734000. 02000000:00000001:1.0:1773941508.002500:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941508.002502:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941508.002504:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941508.002506:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941508.002508:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941508.002511:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941508.002513:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941508.002515:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:1.0:1773941508.002518:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941508.002519:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941508.002521:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941508.002524:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941508.002526:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941508.002530:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0500. 00000020:00000010:1.0:1773941508.002534:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941508.002539:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941508.002543:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941508.002548:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941508.002553:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941508.002558:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941508.002563:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941508.002568:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941508.002572:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941508.002574:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941508.002579:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941508.002584:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941508.002587:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941508.002591:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941508.002595:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941508.002599:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941508.002602:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941508.002604:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941508.002607:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0500. 10000000:01000000:1.0:1773941508.002610:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941508.002634:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941508.002637:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941508.002638:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941508.002640:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941508.002642:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941508.002644:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941508.003677:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941508.003686:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941508.003688:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941508.003691:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941508.003696:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941508.003720:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941508.003728:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941508.003733:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941508.003738:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941508.003749:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607900 00000400:00000200:2.0:1773941508.003753:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941508.003757:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x11cd [1] + 0 00000800:00000001:2.0:1773941508.003763:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941508.003771:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941508.003773:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941508.003777:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941508.003780:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941508.003785:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60e00 x1860112241162496/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941518 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941508.003796:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60e00 x1860112241162496/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941518 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941508.003812:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941508.003817:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941508.003819:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941508.003837:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941508.003839:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941508.003842:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941508.003843:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941508.003845:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941508.003849:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941508.003863:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941508.003865:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941508.003873:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60e00 x1860112241162496/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941518 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941508.003881:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60e00 x1860112241162496/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941518 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941508.003886:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941508.003887:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941508.003889:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941508.003890:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941508.003891:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941508.003893:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941508.003894:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941508.003896:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=87 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941508.003899:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941508.003900:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941508.003901:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941508.003903:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941508.003904:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941508.003906:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941508.003909:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60e00 x1860112241162496/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941518 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941508.003914:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941508.003915:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941508.003916:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941508.003918:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941508.003922:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941508.003923:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941508.003924:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941508.003925:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941508.003927:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941508.003928:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941508.003930:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941508.003933:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941508 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941508.003939:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941508.003944:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941508.003946:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941508.003947:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941508.003948:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941508.003949:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941508.003952:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941508.003954:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941508.003955:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941508.003958:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941508.003960:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941508.003962:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941508.004170:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941508.004176:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941508.004184:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60e00 x1860112241162496/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941518 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941508.004187:0:12230:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:0.0:1773941508.004195:0:12230:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941508.004200:0:12230:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:1.0:1773941508.004203:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241162496:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941508.004214:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941508.004217:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941508.004221:0:12230:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000040:1.0:1773941508.004223:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60e00 x1860112241162496/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941518 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:01000000:0.0:1773941508.004224:0:12230:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941508.004227:0:12230:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941508.004231:0:12230:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941508.004235:0:12230:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941508.004239:0:12230:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:1.0:1773941508.004241:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941508.004241:0:12230:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941508.004244:0:12230:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941508.004246:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 10000000:00000001:0.0:1773941508.004246:0:12230:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941508.004251:0:12230:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:1.0:1773941508.004252:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0000. 00000100:00080000:0.0:1773941508.004254:0:12230:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941508.004258:0:12230:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:1.0:1773941508.004259:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1773941508.004260:0:12230:0:(recover.c:147:ptlrpc_resend()) Process entered 00000020:00000001:1.0:1773941508.004264:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941508.004264:0:12230:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941508.004269:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00080000:0.0:1773941508.004269:0:12230:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941508.004274:0:12230:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000001:1.0:1773941508.004276:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 10000000:01000000:0.0:1773941508.004276:0:12230:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941508.004280:0:12230:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 02000000:00000010:1.0:1773941508.004281:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734c00. 02000000:00000001:1.0:1773941508.004286:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941508.004287:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941508.004290:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60e00. 00000100:00000001:1.0:1773941508.004294:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:0.0:1773941508.004295:0:12230:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:1.0:1773941508.004297:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1773941508.004298:0:12230:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941508.004301:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941508.004306:0:12230:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941508.004309:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:02000000:0.0:1773941508.004312:0:12230:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941508.004313:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941508.004316:0:12230:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941508.004319:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1773941508.004320:0:12230:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941508.004323:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941508.004324:0:12230:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941508.004330:0:12230:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941508.004331:0:12230:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941508.007749:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890eca00. 00010000:00000010:1.1:1773941508.007761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138f600. 00000020:00000001:3.0:1773941508.959925:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941508.959930:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941508.959934:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1773941509.023805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1773941509.023816:0:37:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1773941509.023825:0:37:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021fb80. 00000020:00000040:2.0:1773941509.023832:0:37:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941509.023837:0:37:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941509.023841:0:37:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941509.023846:0:37:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941509.023855:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b98bb0800. 00000100:00000001:2.0:1773941509.023863:0:37:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941509.023875:0:37:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS req@ffff8b5b9021fb80 x1860112241162624/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941509.023887:0:37:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941509.023892:0:37:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941509.023899:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000006259ebb] to pc [ptlrpcd_01_00+0] req@ffff8b5b9021fb80 x1860112241162624/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941509.023924:0:37:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941509.023940:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (284) 00000100:00000001:2.0:1773941509.023954:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941509.023958:0:2354:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941509.023962:0:2354:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941509.023968:0:2354:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021fb80 x1860112241162624/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941509.023977:0:2354:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941509.023980:0:2354:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941509.023984:0:2354:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941509.023988:0:2354:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941509.023991:0:2354:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.023993:0:2355:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1773941509.023996:0:2354:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:1080e6b5-6a6c-4762-b0ec-c785990fec17:2354:1860112241162624:192.168.204.143@tcp:400:kworker.0 00000100:00000001:3.0:1773941509.023999:0:2355:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941509.024004:0:2354:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941509.024011:0:2354:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941509.024014:0:2354:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941509.024017:0:2354:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941509.024023:0:2354:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b90e7f400. 02000000:00000001:2.0:1773941509.024026:0:2354:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941509.024034:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941509.024039:0:2354:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9cb8. 00000400:00000010:2.0:1773941509.024047:0:2354:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34220. 00000100:00000200:2.0:1773941509.024054:0:2354:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241162624, portal 25 00000100:00000001:2.0:1773941509.024058:0:2354:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941509.024060:0:2354:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109271936 : -128249600279680 : ffff8b5b9021fb80) 00000100:00000040:2.0:1773941509.024067:0:2354:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021fb80 x1860112241162624/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941509.024075:0:2354:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941509.024080:0:2354:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941509.024083:0:2354:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34198. 00000100:00000200:2.0:1773941509.024087:0:2354:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241162624, offset 0 00000400:00000200:2.0:1773941509.024094:0:2354:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941509.024104:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941509.024113:0:2354:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941509.024118:0:2354:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941509.024129:0:2354:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941509.024134:0:2354:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941509.024137:0:2354:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941509.024140:0:2354:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[144, 143] 00000400:00000200:2.0:1773941509.024144:0:2354:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:2.0:1773941509.024148:0:2354:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [250:250:256:4294967295] 192.168.204.153@tcp LPNI seq info [145:145:8:4294967295] 00000400:00000200:2.0:1773941509.024153:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941509.024164:0:2354:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:2.0:1773941509.024170:0:2354:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:2.0:1773941509.024175:0:2354:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0a00. 00000800:00000200:2.0:1773941509.024182:0:2354:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:2.0:1773941509.024190:0:2354:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:2.0:1773941509.024194:0:2354:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0a00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1773941509.024210:0:2354:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941509.024215:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941509.024217:0:2354:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294949387 00000100:00000001:2.0:1773941509.024224:0:2354:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294948365 00000100:00000001:2.0:1773941509.024228:0:2354:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294949387 00000100:00000001:2.0:1773941509.024231:0:2354:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294948365 00000100:00000001:2.0:1773941509.024235:0:2354:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941509.024237:0:2354:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941509.024240:0:2354:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941509.024243:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941509.024251:0:2354:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941509.024254:0:2354:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1773941509.024256:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941509.024258:0:2354:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941509.024259:0:2354:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941509.024262:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941509.024406:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941509.024411:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0a00. 00000400:00000200:3.0:1773941509.024416:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941509.024420:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941509.024424:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34198 00000400:00000010:3.0:1773941509.024426:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34198. 00000100:00000001:3.0:1773941509.024429:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941509.024433:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021fb80 x1860112241162624/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.024440:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941509.024443:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021fb80 x1860112241162624/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941525 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.024487:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.024489:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941509.025633:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.025640:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941509.025649:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.025653:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941509.025658:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941509.025663:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941509.025669:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941509.025673:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941509.025678:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941509.025683:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607980 00000400:00000200:2.0:1773941509.025687:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941509.025693:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x11d5 [1] + 224 00000800:00000001:2.0:1773941509.025700:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.025729:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941509.025732:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941509.025735:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941509.025738:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941509.025742:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021fb80 x1860112241162624/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941525 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941509.025750:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b9021fb80 x1860112241162624/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941525 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941509.025763:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941509.025769:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.025771:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941509.025798:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941509.025801:0:2354:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941509.025807:0:2354:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941509.025809:0:2354:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941509.025812:0:2354:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9cb8. 00000400:00000200:3.0:1773941509.025817:0:2354:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34220 00000400:00000010:3.0:1773941509.025819:0:2354:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34220. 00000100:00000001:3.0:1773941509.025822:0:2354:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941509.025828:0:2354:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021fb80 x1860112241162624/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941525 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941509.025837:0:2354:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021fb80 x1860112241162624/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941525 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.025843:0:2354:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941509.025844:0:2354:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941509.025846:0:2354:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941509.025848:0:2354:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941509.025850:0:2354:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941509.025852:0:2354:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941509.025853:0:2354:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941509.025858:0:2354:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=88 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941509.025861:0:2354:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941509.025863:0:2354:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941509.025865:0:2354:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941509.025867:0:2354:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941509.025869:0:2354:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941509.025873:0:2354:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941509.025878:0:2354:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941509.025882:0:2354:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941509.025885:0:2354:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941509.025886:0:2354:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941509.025888:0:2354:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941509.025892:0:2354:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b9021fb80 x1860112241162624/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941525 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.025915:0:2354:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941509.025917:0:2354:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941509.025919:0:2354:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941509.025922:0:2354:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941509.025925:0:2354:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941509.025927:0:2354:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.025929:0:2354:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941509.025931:0:2354:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941509.025932:0:2354:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941509.025934:0:2354:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.025936:0:2354:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941509.025938:0:2354:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941509.025940:0:2354:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941509.025943:0:2354:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941509.025946:0:2354:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 278 00000400:00000200:3.0:1773941509.025951:0:2354:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941509.025954:0:2354:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941509.025958:0:2354:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941509.025960:0:2354:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941509.025962:0:2354:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941509.025963:0:2354:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941509.025966:0:2354:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941509.025967:0:2354:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941509.025969:0:2354:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941509.025971:0:2354:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941509.025973:0:2354:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941509.025974:0:2354:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941509.025976:0:2354:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941509.025977:0:2354:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941509.025978:0:2354:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941509.025980:0:2354:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941509.025982:0:2354:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941509.025991:0:2354:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941509.025993:0:2354:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941509.025995:0:2354:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941509.025997:0:2354:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941509.025999:0:2354:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.026001:0:2354:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941509.026002:0:2354:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941509.026006:0:2354:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275f80. 00000020:00000040:3.0:1773941509.026008:0:2354:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941509.026010:0:2354:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941509.026012:0:2354:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.026014:0:2354:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941509.026015:0:2354:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941509.026017:0:2354:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941509.026020:0:2354:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90262000. 00000100:00000001:3.0:1773941509.026023:0:2354:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941509.026028:0:2354:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90275f80 x1860112241162752/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941509.026033:0:2354:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941509.026035:0:2354:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941509.026038:0:2354:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a88f5799] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90275f80 x1860112241162752/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.026053:0:2354:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.026056:0:2354:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941509.026058:0:2354:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:3.0:1773941509.026063:0:2354:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021fb80 x1860112241162624/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941525 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.026070:0:2354:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941509.026073:0:2354:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941509.026075:0:2354:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941509.026079:0:2354:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021fb80 x1860112241162624/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941525 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941509.026093:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1773941509.026100:0:2354:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:1080e6b5-6a6c-4762-b0ec-c785990fec17:2354:1860112241162624:192.168.204.143@tcp:400:kworker.0 00000100:00000001:1.0:1773941509.026101:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941509.026103:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1773941509.026105:0:2354:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941509.026107:0:2354:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941509.026108:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275f80 x1860112241162752/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941509.026111:0:2354:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021fb80 x1860112241162624/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941525 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941509.026117:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941509.026118:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.026119:0:2354:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941509.026121:0:2354:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1773941509.026121:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941509.026123:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000010:3.0:1773941509.026125:0:2354:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b90e7f400. 02000000:00000001:1.0:1773941509.026126:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941509.026128:0:2354:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:1.0:1773941509.026129:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241162752:192.168.204.143@tcp:250:ptlrpcd_01_00.0 00000020:00000001:3.0:1773941509.026130:0:2354:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941509.026132:0:2354:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941509.026132:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1773941509.026134:0:2354:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:1.0:1773941509.026135:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:3.0:1773941509.026136:0:2354:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b98bb0800. 02000000:00000001:1.0:1773941509.026137:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941509.026139:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1773941509.026140:0:2354:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941509.026142:0:2354:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941509.026143:0:2354:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021fb80. 02000000:00000010:1.0:1773941509.026143:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734c00. 02000000:00000001:1.0:1773941509.026145:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.026147:0:2354:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941509.026148:0:2354:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941509.026149:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941509.026150:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1773941509.026152:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000100:00000001:3.0:1773941509.026154:0:2354:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941509.026156:0:2354:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1773941509.026156:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000001:3.0:1773941509.026158:0:2354:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941509.026159:0:2354:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1773941509.026160:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241162752, portal 25 00000100:00000001:1.0:1773941509.026162:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941509.026164:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109625216 : -128249599926400 : ffff8b5b90275f80) 00000100:00000040:1.0:1773941509.026169:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275f80 x1860112241162752/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941509.026176:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941509.026179:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941509.026182:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1bb0. 00000100:00000200:1.0:1773941509.026185:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241162752, offset 0 00000400:00000200:1.0:1773941509.026190:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941509.026195:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941509.026199:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941509.026203:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941509.026205:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941509.026215:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941509.026217:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941509.026219:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[144, 145] 00000400:00000200:1.0:1773941509.026222:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941509.026225:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [251:251:256:4294967295] 192.168.204.143@tcp LPNI seq info [146:146:8:4294967295] 00000400:00000200:1.0:1773941509.026228:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941509.026233:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941509.026236:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941509.026239:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b200. 00000800:00000200:1.0:1773941509.026243:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941509.026248:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941509.026250:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b200 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:1.0:1773941509.026260:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941509.026265:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941509.026267:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294949389 00000100:00000001:1.0:1773941509.026272:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294949387 00000100:00000001:1.0:1773941509.026276:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294949389 00000100:00000001:1.0:1773941509.026279:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294949387 00000100:00000001:1.0:1773941509.026282:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941509.026284:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941509.026287:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941509.026289:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941509.026293:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941509.026294:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1773941509.026296:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941509.026297:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941509.026299:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941509.026301:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941509.026323:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941509.026327:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b200. 00000400:00000200:3.0:1773941509.026331:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941509.026335:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:3.0:1773941509.026338:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1bb0 00000400:00000010:3.0:1773941509.026341:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1bb0. 00000100:00000001:3.0:1773941509.026344:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941509.026348:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275f80 x1860112241162752/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.026355:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941509.026362:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275f80 x1860112241162752/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.026367:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.026368:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941509.027689:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.027698:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941509.027701:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.027725:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941509.027736:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941509.027743:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941509.027750:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941509.027754:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941509.027759:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941509.027763:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607a00 00000400:00000200:2.0:1773941509.027767:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941509.027778:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x11e5 [1] + 0 00000800:00000001:2.0:1773941509.027783:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.027790:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941509.027792:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941509.027796:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941509.027800:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941509.027806:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275f80 x1860112241162752/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941509.027817:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90275f80 x1860112241162752/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941509.027835:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941509.027840:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.027843:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941509.027866:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941509.027868:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941509.027872:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941509.027873:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941509.027879:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941509.027882:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941509.027884:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941509.027886:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941509.027890:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275f80 x1860112241162752/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941509.027896:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275f80 x1860112241162752/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941509.027902:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941509.027903:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941509.027904:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941509.027906:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941509.027907:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941509.027908:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941509.027910:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941509.027913:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=88 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941509.027915:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941509.027917:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941509.027918:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941509.027920:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941509.027922:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941509.027924:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941509.027926:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275f80 x1860112241162752/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941509.027930:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941509.027932:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941509.027933:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941509.027934:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1773941509.027939:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:1.0:1773941509.027940:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941509.027942:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941509.027943:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941509.027945:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941509.027946:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941509.027951:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:1.0:1773941509.027954:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941509 v=0 (0 0 0 0) 00000100:00001000:1.0:1773941509.027956:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:1.0:1773941509.027961:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:1.0:1773941509.027963:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941509.027964:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:1.0:1773941509.027965:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941509.027966:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941509.027970:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:1.0:1773941509.027971:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:1.0:1773941509.027972:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:1.0:1773941509.027974:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:1.0:1773941509.027977:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:1.0:1773941509.027978:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941509.028154:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941509.028158:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941509.028163:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275f80 x1860112241162752/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941519 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941509.028171:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241162752:192.168.204.143@tcp:250:ptlrpcd_01_00.0 00000100:00000001:1.0:1773941509.028174:0:12232:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:0.0:1773941509.028176:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941509.028179:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1773941509.028184:0:12232:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000040:0.0:1773941509.028185:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275f80 x1860112241162752/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941519 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941509.028189:0:12232:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941509.028191:0:12232:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941509.028193:0:12232:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941509.028197:0:12232:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941509.028200:0:12232:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:0.0:1773941509.028202:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:1.0:1773941509.028204:0:12232:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941509.028207:0:12232:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:0.0:1773941509.028207:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941509.028210:0:12232:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941509.028212:0:12232:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941509.028213:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734c00. 10000000:00000001:1.0:1773941509.028214:0:12232:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941509.028218:0:12232:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:0.0:1773941509.028220:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:1.0:1773941509.028222:0:12232:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000020:00000001:0.0:1773941509.028224:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941509.028225:0:12232:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941509.028229:0:12232:0:(recover.c:147:ptlrpc_resend()) Process entered 00000020:00000040:0.0:1773941509.028230:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941509.028231:0:12232:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941509.028237:0:12232:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000020:00000001:0.0:1773941509.028237:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941509.028241:0:12232:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:0.0:1773941509.028242:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90262000. 10000000:01000000:1.0:1773941509.028243:0:12232:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 02000000:00000001:0.0:1773941509.028249:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000040:1.0:1773941509.028250:0:12232:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 02000000:00000001:0.0:1773941509.028253:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941509.028257:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275f80. 00000100:00000001:0.0:1773941509.028262:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941509.028264:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941509.028265:0:12232:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:0.0:1773941509.028267:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941509.028269:0:12232:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941509.028272:0:12232:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941509.028275:0:12232:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941509.028279:0:12232:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941509.028283:0:12232:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941509.028285:0:12232:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941509.028288:0:12232:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941509.028289:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941509.028290:0:12232:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941509.028293:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941509.028297:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941509.028306:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941509.791830:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941509.791838:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941509.791843:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941509.791845:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941509.791847:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941509.791849:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941509.791852:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941509.791854:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941509.791856:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941509.791859:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941509.791861:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941509.791873:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941509.791879:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941509.791884:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941509.791888:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941509.791894:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941509.791914:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2600. 00000100:00000001:3.0:1773941509.791921:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941509.791926:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941509.791930:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941509.791937:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962800. 00010000:00000001:3.0:1773941509.791944:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941509.791948:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941509.791953:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941509.791955:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1de1 to hash 00000020:00000001:3.0:1773941509.791958:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941509.791959:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941509.791963:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941509.791967:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941509.791970:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941509.791976:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941509.792005:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941509.792017:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941509.792024:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941509.792026:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941509.792033:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941509.792035:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941509.792038:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941509.792040:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941509.792045:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941509.792050:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.792053:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941509.792055:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941509.792071:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241162880/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.792081:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941509.792084:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941509.792087:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941509.792090:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941509.792092:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941509.792096:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241162880:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941509.792100:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941509.792106:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941509.792108:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941509.792111:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941509.792116:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90263000. 02000000:00000001:3.0:1773941509.792119:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941509.792127:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941509.792136:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941509.792144:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941509.792151:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241162880, portal 25 00000100:00000001:3.0:1773941509.792153:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941509.792154:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941509.792161:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241162880/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941525 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.792168:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941509.792171:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941509.792174:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6ae58. 00000100:00000200:3.0:1773941509.792178:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241162880, offset 0 00000400:00000200:3.0:1773941509.792184:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941509.792193:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941509.792200:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941509.792206:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941509.792210:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941509.792214:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941509.792217:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941509.792220:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[146, 145] 00000400:00000200:3.0:1773941509.792225:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941509.792229:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [252:252:256:4294967295] 192.168.204.153@tcp LPNI seq info [147:147:8:4294967295] 00000400:00000200:3.0:1773941509.792234:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941509.792244:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941509.792251:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941509.792257:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69c00. 00000800:00000200:3.0:1773941509.792264:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941509.792271:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941509.792276:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69c00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941509.792298:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941509.792302:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941509.792305:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294950155 00000100:00000001:3.0:1773941509.792317:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294949389 00000100:00000001:3.0:1773941509.792324:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294950155 00000100:00000001:3.0:1773941509.792328:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294949389 00000100:00000001:3.0:1773941509.792333:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941509.792336:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.792339:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941509.792341:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941509.792344:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941509.792347:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941509.792350:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941509.792417:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941509.792422:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69c00. 00000400:00000200:2.0:1773941509.792427:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941509.792431:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941509.792435:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6ae58 00000400:00000010:2.0:1773941509.792437:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6ae58. 00000100:00000001:2.0:1773941509.792441:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941509.792445:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241162880/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941525 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941509.792454:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941509.792456:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241162880/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941525 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941509.792461:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941509.792463:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941509.793630:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.793642:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941509.793645:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.793647:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941509.793653:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941509.793658:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941509.793672:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941509.793677:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941509.793681:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941509.793686:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607a80 00000400:00000200:2.0:1773941509.793690:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941509.793696:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x11ed [1] + 224 00000800:00000001:2.0:1773941509.793737:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.793746:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941509.793749:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941509.793753:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941509.793757:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941509.793763:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241162880/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941525 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941509.793773:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241162880/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941525 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941509.793792:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941509.793799:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.793804:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941509.793813:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941509.793819:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941509.793821:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941509.793823:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941509.793827:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941509.793829:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941509.793831:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941509.793836:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241162880/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941525 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941509.793844:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241162880/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941525 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.793849:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941509.793850:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941509.793852:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941509.793854:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941509.793856:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941509.793857:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941509.793859:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941509.793863:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=88 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941509.793867:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941509.793870:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941509.793874:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941509.793877:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941509.793887:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941509.793891:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941509.793895:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941509.793900:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941509.793904:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941509.793906:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941509.793908:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941509.793911:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941509.793915:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241162880/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941525 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.793924:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941509.793925:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941509.793927:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941509.793930:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941509.793932:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941509.793933:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.793935:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941509.793936:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941509.793936:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941509.793937:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.793939:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941509.793940:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941509.793942:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941509.793944:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941509.793946:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 279 00000400:00000200:3.0:1773941509.793951:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941509.793956:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941509.793960:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941509.793963:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941509.793967:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941509.793969:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941509.793972:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941509.793980:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941509.793983:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941509.793985:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941509.793988:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941509.793991:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941509.793995:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941509.793997:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941509.793999:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941509.794001:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941509.794004:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941509.794008:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941509.794010:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941509.794012:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941509.794014:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941509.794016:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.794018:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941509.794019:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941509.794023:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275f80. 00000020:00000040:3.0:1773941509.794025:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941509.794027:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941509.794028:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.794030:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941509.794031:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941509.794033:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941509.794036:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90260c00. 00000100:00000001:3.0:1773941509.794039:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941509.794043:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90275f80 x1860112241163008/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941509.794050:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941509.794052:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941509.794055:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a88f5799] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90275f80 x1860112241163008/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.794079:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.794081:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941509.794083:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941509.794085:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941509.794086:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941509.794087:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941509.794089:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941509.794090:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941509.794092:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941509.794095:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241162880/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941525 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.794101:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941509.794103:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941509.794104:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941509.794106:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941509.794110:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241162880/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941525 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941509.794116:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241162880:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941509.794119:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941509.794121:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941509.794122:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941509.794124:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941509.794127:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941509.794129:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241162880/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941525 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941509.794129:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941509.794133:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1773941509.794140:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941509.794142:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941509.794144:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000040:0.0:1773941509.794145:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275f80 x1860112241163008/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.794146:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:3.0:1773941509.794151:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:3.0:1773941509.794158:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941509.794160:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1773941509.794162:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1773941509.794163:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 00000020:00000001:3.0:1773941509.794168:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000100:00000001:0.0:1773941509.794168:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941509.794175:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 02000000:00000001:0.0:1773941509.794177:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00010000:3.0:1773941509.794178:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de1 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941509.794182:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1773941509.794186:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:0.0:1773941509.794188:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941509.794191:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de1 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1773941509.794196:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241163008:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:3.0:1773941509.794198:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:3.0:1773941509.794201:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de1 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941509.794205:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:3.0:1773941509.794209:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000001:0.0:1773941509.794211:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:3.0:1773941509.794212:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:3.0:1773941509.794215:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de1 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941509.794216:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941509.794223:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:3.0:1773941509.794228:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941509.794230:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941509.794231:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0c00. 00010000:00000001:3.0:1773941509.794234:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 02000000:00000001:0.0:1773941509.794236:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941509.794239:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de1 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941509.794246:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941509.794252:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00010000:3.0:1773941509.794254:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:3.0:1773941509.794259:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000010:0.0:1773941509.794260:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 10000000:00010000:3.0:1773941509.794263:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de1 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:0.0:1773941509.794266:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241163008, portal 25 00000100:00000001:0.0:1773941509.794270:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941509.794272:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109625216 : -128249599926400 : ffff8b5b90275f80) 00000100:00000040:0.0:1773941509.794278:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275f80 x1860112241163008/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941509.794292:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941509.794296:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941509.794300:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c550. 00000100:00000200:0.0:1773941509.794306:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241163008, offset 0 00010000:00000001:3.0:1773941509.794313:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:3.0:1773941509.794317:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:0.0:1773941509.794318:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000020:00000001:3.0:1773941509.794320:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941509.794324:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000400:00000200:0.0:1773941509.794326:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:3.0:1773941509.794328:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941509.794333:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000200:0.0:1773941509.794334:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:3.0:1773941509.794337:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:3.0:1773941509.794340:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de1 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941509.794350:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00000001:3.0:1773941509.794353:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941509.794356:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 10000000:00010000:3.0:1773941509.794358:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de1 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941509.794363:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941509.794370:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:01000000:3.0:1773941509.794372:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:0.0:1773941509.794378:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[146, 147] 10000000:00000040:3.0:1773941509.794379:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941509.794382:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941509.794387:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00010000:3.0:1773941509.794389:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941509.794392:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:0.0:1773941509.794394:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [253:253:256:4294967295] 192.168.204.143@tcp LPNI seq info [148:148:8:4294967295] 00010000:00000001:3.0:1773941509.794397:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941509.794401:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:0.0:1773941509.794405:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941509.794406:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941509.794410:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941509.794414:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1de1 from hash 00010000:00000001:3.0:1773941509.794420:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:0.0:1773941509.794421:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:3.0:1773941509.794422:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941509.794424:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941509.794427:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:3.0:1773941509.794429:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941509.794431:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:3.0:1773941509.794436:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:0.0:1773941509.794436:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941509.794438:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941509.794440:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1773941509.794444:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 10000000:00000001:3.0:1773941509.794450:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941509.794454:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de1 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941509.794454:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:3.0:1773941509.794463:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941509.794465:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941509.794467:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000800:00000200:0.0:1773941509.794467:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:3.0:1773941509.794470:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941509.794473:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:0.0:1773941509.794474:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:3.0:1773941509.794475:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941509.794476:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941509.794479:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de1 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:3.0:1773941509.794485:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962800 count: 0 00000100:00000001:0.0:1773941509.794491:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941509.794494:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941509.794496:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:0.0:1773941509.794500:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941509.794501:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941509.794506:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294950157 00000100:00000040:3.0:1773941509.794508:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241162880/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941525 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941509.794514:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294950155 00000100:00000001:3.0:1773941509.794516:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941509.794521:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1773941509.794523:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294950157 02000000:00000010:3.0:1773941509.794524:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90263000. 02000000:00000001:3.0:1773941509.794528:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941509.794530:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941509.794532:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941509.794532:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294950155 00000020:00000001:3.0:1773941509.794536:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941509.794540:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2600. 00000100:00000001:0.0:1773941509.794541:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941509.794543:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941509.794547:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000800:00000200:2.0:1773941509.794548:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1773941509.794548:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941509.794549:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:0.0:1773941509.794553:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1773941509.794554:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000100:00000001:3.0:1773941509.794555:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941509.794558:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941509.794559:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941509.794561:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941509.794561:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:01000000:3.0:1773941509.794566:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:0.0:1773941509.794569:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000001:3.0:1773941509.794571:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000400:00000200:2.0:1773941509.794571:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:0.0:1773941509.794573:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:2.0:1773941509.794576:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c550 00000100:00000001:0.0:1773941509.794578:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:2.0:1773941509.794579:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c550. 00000100:00000001:0.0:1773941509.794592:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000010:3.0:1773941509.794594:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068f40. 00000100:00000001:0.0:1773941509.794595:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941509.794599:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1773941509.794601:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941509.794602:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b839e2600. 00000100:00000200:2.0:1773941509.794606:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275f80 x1860112241163008/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1773941509.794609:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b839e3a00. 00000040:00000010:3.0:1773941509.794615:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b839e3000. 00000100:00000001:2.0:1773941509.794617:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000010:3.0:1773941509.794620:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000100:00000040:2.0:1773941509.794622:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275f80 x1860112241163008/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941509.794629:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941509.794636:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:3.0:1773941509.794685:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b839e2a00. 80000000:00000010:3.0:1773941509.794690:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941509.794694:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b84d8ae00. 20000000:00000010:3.0:1773941509.794698:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b84d8bc00. 10000000:00000040:3.0:1773941509.794701:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:3.0:1773941509.794728:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941509.794732:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b839e3a00. 00000040:00000010:3.0:1773941509.794736:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b839e3000. 00000020:00000010:3.0:1773941509.794738:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941509.794741:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b839e2a00. 80000000:00000010:3.0:1773941509.794743:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941509.794746:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b84d8ae00. 20000000:00000010:3.0:1773941509.794749:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b84d8bc00. 00000020:00000010:3.0:1773941509.794752:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b839e2600. 10000000:00000010:3.0:1773941509.794755:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068f40. 10000000:01000000:3.0:1773941509.794758:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941509.794760:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941509.794762:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941509.794764:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941509.794766:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941509.794767:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941509.794769:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941509.795905:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.795912:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941509.795915:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.795917:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941509.795922:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941509.795927:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941509.795932:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941509.795936:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941509.795938:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941509.795940:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607b00 00000400:00000200:2.0:1773941509.795943:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941509.795946:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x11fd [1] + 0 00000800:00000001:2.0:1773941509.795950:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.795954:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941509.795956:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941509.795959:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941509.795962:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941509.795965:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275f80 x1860112241163008/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941509.795972:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90275f80 x1860112241163008/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941509.795985:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941509.795989:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941509.795991:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941509.796018:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941509.796026:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941509.796029:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941509.796030:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941509.796032:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941509.796035:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941509.796037:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941509.796039:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941509.796043:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275f80 x1860112241163008/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941509.796051:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275f80 x1860112241163008/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941509.796057:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941509.796058:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941509.796059:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941509.796060:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941509.796062:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941509.796063:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941509.796065:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941509.796067:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=88 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941509.796070:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941509.796071:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941509.796072:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941509.796074:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941509.796075:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941509.796077:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941509.796080:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275f80 x1860112241163008/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941509.796085:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941509.796086:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941509.796087:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941509.796089:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941509.796093:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941509.796094:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941509.796095:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941509.796096:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941509.796101:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941509.796103:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941509.796105:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941509.796108:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941509 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941509.796110:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941509.796114:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941509.796115:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941509.796117:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941509.796118:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941509.796119:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941509.796122:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941509.796123:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941509.796125:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941509.796127:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941509.796130:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941509.796131:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941509.796340:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941509.796344:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941509.796349:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275f80 x1860112241163008/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941519 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941509.796358:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241163008:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941509.796359:0:12233:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:1.0:1773941509.796363:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941509.796366:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1773941509.796368:0:12233:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000040:1.0:1773941509.796372:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275f80 x1860112241163008/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941519 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941509.796373:0:12233:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941509.796377:0:12233:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941509.796380:0:12233:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:1.0:1773941509.796384:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941509.796384:0:12233:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:1.0:1773941509.796387:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941509.796388:0:12233:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:1.0:1773941509.796389:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0c00. 00000001:00000001:0.0:1773941509.796391:0:12233:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941509.796393:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941509.796394:0:12233:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:1.0:1773941509.796395:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941509.796396:0:12233:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:1.0:1773941509.796398:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941509.796399:0:12233:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941509.796402:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000001:0.0:1773941509.796403:0:12233:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941509.796405:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90260c00. 00000100:00000001:0.0:1773941509.796406:0:12233:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:1.0:1773941509.796410:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:0.0:1773941509.796411:0:12233:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:1.0:1773941509.796412:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941509.796415:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275f80. 00000100:00000001:0.0:1773941509.796416:0:12233:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941509.796419:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941509.796419:0:12233:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941509.796421:0:12233:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941509.796422:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941509.796424:0:12233:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941509.796425:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941509.796432:0:12233:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941509.796435:0:12233:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941509.796436:0:12233:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:1.0:1773941509.796437:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941509.796439:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941509.796441:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941509.796442:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941509.796449:0:12233:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941509.796451:0:12233:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941509.796453:0:12233:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941509.796457:0:12233:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941509.796460:0:12233:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941509.796462:0:12233:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941509.796463:0:12233:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941509.796465:0:12233:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941509.796467:0:12233:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941509.799741:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962800. 00010000:00000010:3.1:1773941509.799749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:0.0:1773941509.983851:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941509.983856:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941509.983860:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941511.007882:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941511.007891:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941511.007894:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941511.647830:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941511.647840:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941511.647845:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941511.647847:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941511.647870:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941511.647871:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941511.647875:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941511.647877:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941511.647880:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941511.647882:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941511.647883:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941511.647892:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941511.647897:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941511.647902:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941511.647907:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941511.647912:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941511.647918:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b84d8ae00. 00000100:00000001:3.0:1773941511.647926:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941511.647930:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941511.647933:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941511.647940:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962800. 00010000:00000001:3.0:1773941511.647946:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941511.647949:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941511.647953:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941511.647955:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1de8 to hash 00000020:00000001:3.0:1773941511.647957:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941511.647958:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941511.647961:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941511.647965:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941511.647968:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941511.647971:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941511.647980:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941511.647992:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941511.648004:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941511.648006:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941511.648012:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941511.648014:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941511.648016:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941511.648018:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941511.648023:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941511.648028:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.648029:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941511.648031:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941511.648041:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241163136/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941511.648048:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941511.648050:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941511.648053:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941511.648056:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941511.648058:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941511.648060:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241163136:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941511.648063:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941511.648069:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941511.648071:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941511.648074:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941511.648077:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941511.648078:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941511.648086:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941511.648089:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941511.648096:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941511.648102:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241163136, portal 25 00000100:00000001:3.0:1773941511.648104:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941511.648105:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941511.648118:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241163136/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941527 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941511.648133:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941511.648135:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941511.648137:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6aaa0. 00000100:00000200:3.0:1773941511.648140:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241163136, offset 0 00000400:00000200:3.0:1773941511.648146:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941511.648155:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941511.648163:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941511.648168:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941511.648172:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941511.648175:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941511.648178:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941511.648180:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[148, 147] 00000400:00000200:3.0:1773941511.648183:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941511.648187:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [254:254:256:4294967295] 192.168.204.153@tcp LPNI seq info [149:149:8:4294967295] 00000400:00000200:3.0:1773941511.648190:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941511.648198:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941511.648203:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941511.648207:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69700. 00000800:00000200:3.0:1773941511.648214:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941511.648221:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941511.648226:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69700 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941511.648242:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941511.648245:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941511.648248:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294952011 00000100:00000001:3.0:1773941511.648255:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294950157 00000100:00000001:3.0:1773941511.648260:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294952011 00000100:00000001:3.0:1773941511.648263:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294950157 00000100:00000001:3.0:1773941511.648265:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941511.648273:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.648280:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941511.648285:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941511.648289:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941511.648292:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941511.648296:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941511.648414:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941511.648425:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69700. 00000400:00000200:2.0:1773941511.648435:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941511.648443:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941511.648451:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6aaa0 00000400:00000010:2.0:1773941511.648454:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6aaa0. 00000100:00000001:2.0:1773941511.648460:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941511.648468:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241163136/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941527 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941511.648492:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941511.648495:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241163136/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941527 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941511.648501:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941511.648502:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941511.649492:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941511.649511:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941511.649513:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941511.649515:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941511.649521:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941511.649525:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941511.649531:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941511.649534:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941511.649539:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941511.649542:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607b80 00000400:00000200:2.0:1773941511.649559:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941511.649564:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x1205 [1] + 224 00000800:00000001:2.0:1773941511.649573:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941511.649582:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941511.649585:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941511.649588:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941511.649593:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941511.649597:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241163136/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941527 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941511.649608:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241163136/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941527 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941511.649632:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941511.649641:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941511.649644:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941511.649721:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941511.649729:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941511.649730:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941511.649733:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941511.649737:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941511.649738:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941511.649742:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941511.649747:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241163136/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941527 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941511.649756:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241163136/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941527 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941511.649762:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941511.649764:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941511.649766:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941511.649768:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941511.649770:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941511.649772:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941511.649775:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941511.649780:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=90 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941511.649783:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941511.649785:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941511.649789:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941511.649791:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941511.649794:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941511.649796:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941511.649800:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941511.649804:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941511.649807:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941511.649816:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941511.649817:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941511.649819:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941511.649822:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241163136/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941527 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941511.649828:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941511.649829:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941511.649830:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941511.649833:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941511.649835:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941511.649836:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.649838:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941511.649838:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941511.649839:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941511.649840:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.649841:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941511.649844:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941511.649845:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941511.649848:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941511.649850:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 280 00000400:00000200:3.0:1773941511.649854:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941511.649858:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941511.649861:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941511.649864:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941511.649866:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941511.649868:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941511.649870:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941511.649872:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941511.649874:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941511.649876:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941511.649878:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941511.649879:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941511.649881:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941511.649885:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941511.649887:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941511.649889:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941511.649891:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941511.649894:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941511.649895:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941511.649897:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941511.649900:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941511.649902:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.649904:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941511.649905:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941511.649908:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90276300. 00000020:00000040:3.0:1773941511.649910:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941511.649912:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941511.649913:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.649915:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941511.649917:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941511.649919:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941511.649922:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90263000. 00000100:00000001:3.0:1773941511.649926:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941511.649930:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90276300 x1860112241163264/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941511.649937:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941511.649939:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941511.649942:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006af361f1] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90276300 x1860112241163264/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941511.649960:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.649962:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941511.649964:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941511.649966:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941511.649968:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941511.649969:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941511.649974:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941511.649976:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941511.649977:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941511.649981:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241163136/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941527 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941511.649986:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941511.649989:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941511.649991:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941511.649993:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941511.649997:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241163136/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941527 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941511.650002:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241163136:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941511.650006:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941511.650008:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941511.650010:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941511.650012:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941511.650013:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941511.650018:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241163136/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941527 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941511.650021:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941511.650030:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1773941511.650031:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941511.650035:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941511.650038:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941511.650040:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:3.0:1773941511.650045:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000040:1.0:1773941511.650045:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90276300 x1860112241163264/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941511.650048:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941511.650051:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941511.650057:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 00000020:00000001:3.0:1773941511.650065:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941511.650072:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00010000:3.0:1773941511.650079:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de8 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941511.650081:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941511.650086:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941511.650096:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941511.650105:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1773941511.650108:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:1.0:1773941511.650112:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941511.650114:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de8 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941511.650122:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00100000:1.0:1773941511.650123:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241163264:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00010000:3.0:1773941511.650126:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de8 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941511.650140:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:3.0:1773941511.650141:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941511.650149:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 02000000:00000001:1.0:1773941511.650151:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00010000:3.0:1773941511.650152:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de8 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:1.0:1773941511.650158:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941511.650160:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941511.650164:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941511.650166:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:3.0:1773941511.650168:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:3.0:1773941511.650172:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de8 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:1.0:1773941511.650175:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734000. 00010000:00010000:3.0:1773941511.650178:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 02000000:00000001:1.0:1773941511.650182:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941511.650183:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941511.650187:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de8 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941511.650192:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:3.0:1773941511.650193:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941511.650194:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941511.650195:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000020:00000001:3.0:1773941511.650196:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941511.650198:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000200:1.0:1773941511.650207:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941511.650209:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:3.0:1773941511.650214:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de8 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:1.0:1773941511.650217:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 10000000:00000001:3.0:1773941511.650226:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941511.650229:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de8 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:1.0:1773941511.650231:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 10000000:01000000:3.0:1773941511.650242:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:3.0:1773941511.650245:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941511.650247:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1773941511.650247:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241163264, portal 25 00010000:00010000:3.0:1773941511.650250:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941511.650254:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:1.0:1773941511.650255:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:3.0:1773941511.650258:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:1.0:1773941511.650259:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109626112 : -128249599925504 : ffff8b5b90276300) 00010000:00000001:3.0:1773941511.650261:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941511.650263:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941511.650265:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941511.650269:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1de8 from hash 00010000:00000001:3.0:1773941511.650271:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941511.650275:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941511.650277:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000040:1.0:1773941511.650278:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90276300 x1860112241163264/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941521 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941511.650283:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:3.0:1773941511.650285:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941511.650287:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:3.0:1773941511.650291:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941511.650294:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941511.650296:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941511.650301:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941511.650309:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de8 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941511.650324:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:1.0:1773941511.650325:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:3.0:1773941511.650326:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941511.650329:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941511.650332:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941511.650341:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000040:1.0:1773941511.650341:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941511.650346:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941511.650348:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941511.650350:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1de8 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:1.0:1773941511.650351:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1990. 00010000:00000040:3.0:1773941511.650364:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962800 count: 0 00000100:00000200:1.0:1773941511.650365:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241163264, offset 0 00000400:00000200:1.0:1773941511.650378:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941511.650381:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941511.650383:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941511.650385:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941511.650392:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241163136/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941527 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941511.650400:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000400:00000200:1.0:1773941511.650401:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 02000000:00000001:3.0:1773941511.650404:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941511.650408:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941511.650411:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941511.650417:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:1.0:1773941511.650417:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000020:00000040:3.0:1773941511.650419:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941511.650428:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941511.650431:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b84d8ae00. 00000400:00000200:1.0:1773941511.650435:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1773941511.650437:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941511.650443:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941511.650445:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000400:00000200:1.0:1773941511.650448:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000100:00000001:3.0:1773941511.650451:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941511.650452:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941511.650454:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941511.650456:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941511.650458:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941511.650462:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068280. 00000020:00000010:3.0:1773941511.650465:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84d8ae00. 00000400:00000200:1.0:1773941511.650469:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000020:00000010:3.0:1773941511.650473:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b84d8bc00. 00000400:00000200:1.0:1773941511.650479:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000040:00000010:3.0:1773941511.650480:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b84d8a200. 00000020:00000010:3.0:1773941511.650489:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000400:00000200:1.0:1773941511.650491:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[148, 149] 00000020:00000010:3.0:1773941511.650496:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b839e2600. 00000400:00000200:1.0:1773941511.650501:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 80000000:00000010:3.0:1773941511.650502:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00000400:00000200:1.0:1773941511.650512:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [255:255:256:4294967295] 192.168.204.143@tcp LPNI seq info [150:150:8:4294967295] 00040000:00000010:3.0:1773941511.650515:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b839e2a00. 20000000:00000010:3.0:1773941511.650524:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b839e3000. 00000400:00000200:1.0:1773941511.650533:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000040:3.0:1773941511.650536:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:3.0:1773941511.650541:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941511.650563:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b84d8bc00. 00000400:00000200:1.0:1773941511.650564:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000040:00000010:3.0:1773941511.650572:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b84d8a200. 00000020:00000010:3.0:1773941511.650577:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941511.650580:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b839e2600. 00000800:00000200:1.0:1773941511.650582:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 80000000:00000010:3.0:1773941511.650584:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941511.650591:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b839e2a00. 00000800:00000010:1.0:1773941511.650593:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b900. 20000000:00000010:3.0:1773941511.650594:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b839e3000. 00000020:00000010:3.0:1773941511.650603:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84d8ae00. 10000000:00000010:3.0:1773941511.650606:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068280. 00000800:00000200:1.0:1773941511.650612:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:01000000:3.0:1773941511.650614:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941511.650622:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941511.650627:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000800:00000200:1.0:1773941511.650631:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 10000000:00000001:3.0:1773941511.650632:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941511.650636:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941511.650646:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 00000800:00000200:1.0:1773941511.650648:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b900 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:3.0:1773941511.650650:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:1.0:1773941511.650671:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941511.650676:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941511.650679:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294952013 00000100:00000001:1.0:1773941511.650685:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294952011 00000100:00000001:1.0:1773941511.650691:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294952013 00000100:00000001:1.0:1773941511.650697:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294952011 00000100:00000001:1.0:1773941511.650701:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941511.650744:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941511.650748:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941511.650752:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941511.650763:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941511.650765:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1773941511.650768:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941511.650771:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941511.650774:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941511.650782:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1773941511.650784:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1773941511.650795:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b900. 00000400:00000200:2.0:1773941511.650799:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941511.650803:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941511.650806:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1990 00000400:00000010:2.0:1773941511.650808:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1990. 00000100:00000001:2.0:1773941511.650811:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941511.650816:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90276300 x1860112241163264/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941521 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941511.650823:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941511.650826:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90276300 x1860112241163264/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941521 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941511.650834:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941511.650836:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941511.652276:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941511.652291:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941511.652295:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941511.652298:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941511.652305:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941511.652313:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941511.652321:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941511.652326:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941511.652331:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941511.652336:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607c00 00000400:00000200:2.0:1773941511.652341:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941511.652347:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1215 [1] + 0 00000800:00000001:2.0:1773941511.652355:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941511.652363:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941511.652367:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941511.652372:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941511.652378:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941511.652385:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90276300 x1860112241163264/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941521 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941511.652398:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90276300 x1860112241163264/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941521 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941511.652431:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941511.652439:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941511.652442:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941511.652479:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941511.652483:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941511.652489:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941511.652491:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941511.652495:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941511.652500:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941511.652502:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941511.652505:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941511.652512:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90276300 x1860112241163264/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941521 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941511.652524:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90276300 x1860112241163264/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941521 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941511.652533:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941511.652535:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941511.652537:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941511.652540:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941511.652543:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941511.652559:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941511.652563:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941511.652570:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=90 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941511.652574:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941511.652578:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941511.652580:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941511.652583:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941511.652585:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941511.652588:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941511.652593:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90276300 x1860112241163264/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941521 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941511.652601:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941511.652604:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941511.652607:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941511.652610:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1773941511.652616:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:1.0:1773941511.652619:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941511.652629:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941511.652632:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941511.652634:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941511.652637:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941511.652640:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:1.0:1773941511.652644:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941511 v=0 (0 0 0 0) 00000100:00001000:1.0:1773941511.652647:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:1.0:1773941511.652654:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:1.0:1773941511.652657:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941511.652660:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:1.0:1773941511.652662:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941511.652664:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941511.652668:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:1.0:1773941511.652671:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:1.0:1773941511.652674:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:1.0:1773941511.652677:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:1.0:1773941511.652681:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:1.0:1773941511.652684:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941511.652951:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941511.652955:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941511.652960:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90276300 x1860112241163264/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941521 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941511.652967:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241163264:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941511.652973:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941511.652976:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941511.652979:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90276300 x1860112241163264/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941521 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941511.652984:0:12234:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:0.0:1773941511.652984:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941511.652989:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941511.652992:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734000. 02000000:00000001:0.0:1773941511.652995:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:1.0:1773941511.652996:0:12234:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000020:00000001:0.0:1773941511.652998:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00100000:1.0:1773941511.653002:0:12234:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000040:0.0:1773941511.653002:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941511.653007:0:12234:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000001:0.0:1773941511.653008:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 10000000:01000000:1.0:1773941511.653012:0:12234:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000010:0.0:1773941511.653013:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90263000. 00000001:00000001:1.0:1773941511.653017:0:12234:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:0.0:1773941511.653018:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:1.0:1773941511.653022:0:12234:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:0.0:1773941511.653026:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941511.653029:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90276300. 00000001:00000001:1.0:1773941511.653030:0:12234:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941511.653035:0:12234:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:0.0:1773941511.653035:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941511.653039:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941511.653040:0:12234:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:0.0:1773941511.653042:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941511.653044:0:12234:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941511.653047:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941511.653050:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941511.653052:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941511.653053:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941511.653059:0:12234:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941511.653064:0:12234:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941511.653069:0:12234:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941511.653072:0:12234:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941511.653081:0:12234:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941511.653083:0:12234:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941511.653086:0:12234:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941511.653088:0:12234:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941511.653090:0:12234:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941511.653092:0:12234:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941511.653110:0:12234:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941511.653112:0:12234:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941511.653114:0:12234:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941511.653118:0:12234:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941511.653123:0:12234:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941511.653125:0:12234:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941511.653127:0:12234:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941511.653131:0:12234:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941511.653133:0:12234:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941511.655736:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962800. 00010000:00000010:3.1:1773941511.655744:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:0.0:1773941512.031826:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941512.031836:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941512.031841:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941513.055882:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941513.055890:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941513.055895:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941513.439797:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941513.439803:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941513.439807:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941513.439809:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941513.439810:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941513.439811:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941513.439814:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941513.439815:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941513.439817:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941513.439820:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941513.439821:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941513.439831:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941513.439849:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941513.439853:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941513.439855:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941513.439860:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941513.439866:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2600. 00000100:00000001:3.0:1773941513.439871:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941513.439875:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941513.439878:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941513.439885:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962800. 00010000:00000001:3.0:1773941513.439889:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941513.439891:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941513.439894:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941513.439897:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1def to hash 00000020:00000001:3.0:1773941513.439899:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941513.439900:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941513.439902:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941513.439905:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941513.439907:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941513.439910:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1def lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941513.439918:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1def lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941513.439927:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1def lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941513.439932:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941513.439933:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941513.439938:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941513.439940:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941513.439942:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941513.439943:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941513.439947:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941513.439954:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941513.439956:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941513.439957:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941513.439966:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241163392/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941513.439973:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941513.439975:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941513.439978:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941513.439980:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941513.439982:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941513.439984:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241163392:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941513.439987:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941513.439997:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941513.439999:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941513.440001:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941513.440005:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941513.440007:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941513.440012:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941513.440015:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941513.440021:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941513.440026:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241163392, portal 25 00000100:00000001:3.0:1773941513.440028:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941513.440029:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941513.440033:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241163392/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941529 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941513.440038:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941513.440041:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941513.440042:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6add0. 00000100:00000200:3.0:1773941513.440044:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241163392, offset 0 00000400:00000200:3.0:1773941513.440049:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941513.440055:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941513.440062:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941513.440066:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941513.440073:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941513.440076:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941513.440078:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941513.440081:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[150, 149] 00000400:00000200:3.0:1773941513.440084:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941513.440087:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [256:256:256:4294967295] 192.168.204.153@tcp LPNI seq info [151:151:8:4294967295] 00000400:00000200:3.0:1773941513.440090:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941513.440099:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941513.440104:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941513.440109:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69f00. 00000800:00000200:3.0:1773941513.440116:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941513.440124:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941513.440130:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69f00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941513.440149:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941513.440154:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941513.440158:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294953803 00000100:00000001:3.0:1773941513.440165:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294952013 00000100:00000001:3.0:1773941513.440169:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294953803 00000100:00000001:3.0:1773941513.440172:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294952013 00000100:00000001:3.0:1773941513.440174:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941513.440176:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941513.440178:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941513.440180:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941513.440182:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941513.440185:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941513.440187:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941513.440269:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941513.440275:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69f00. 00000400:00000200:2.0:1773941513.440280:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941513.440287:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941513.440292:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6add0 00000400:00000010:2.0:1773941513.440295:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6add0. 00000100:00000001:2.0:1773941513.440299:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941513.440305:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241163392/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941529 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941513.440316:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941513.440321:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241163392/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941529 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941513.440329:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941513.440331:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941513.441304:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941513.441315:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941513.441318:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941513.441321:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941513.441326:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941513.441332:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941513.441339:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941513.441343:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941513.441347:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941513.441351:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607c80 00000400:00000200:2.0:1773941513.441356:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941513.441361:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x121d [1] + 224 00000800:00000001:2.0:1773941513.441368:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941513.441377:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941513.441380:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941513.441392:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941513.441397:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941513.441403:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241163392/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941529 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941513.441414:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241163392/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941529 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941513.441432:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941513.441441:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941513.441446:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941513.441455:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941513.441461:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941513.441463:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941513.441465:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941513.441468:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941513.441474:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941513.441477:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941513.441481:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241163392/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941529 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941513.441489:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241163392/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941529 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941513.441494:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941513.441495:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941513.441497:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941513.441499:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941513.441501:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941513.441502:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941513.441504:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941513.441508:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=92 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941513.441510:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941513.441525:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941513.441529:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941513.441531:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941513.441533:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941513.441536:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941513.441539:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941513.441542:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941513.441545:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941513.441546:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941513.441547:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941513.441549:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941513.441551:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241163392/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941529 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941513.441556:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941513.441557:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941513.441559:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941513.441565:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941513.441566:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941513.441568:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941513.441569:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941513.441570:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941513.441571:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941513.441572:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941513.441573:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941513.441574:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941513.441576:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941513.441578:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941513.441580:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 282 00000400:00000200:3.0:1773941513.441583:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941513.441586:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941513.441589:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941513.441592:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941513.441594:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941513.441595:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941513.441598:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941513.441599:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941513.441600:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941513.441602:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941513.441604:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941513.441605:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941513.441607:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941513.441609:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941513.441610:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941513.441611:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941513.441613:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941513.441616:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941513.441617:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941513.441619:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941513.441621:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941513.441625:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941513.441627:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941513.441629:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941513.441631:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275500. 00000020:00000040:3.0:1773941513.441633:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941513.441635:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941513.441637:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941513.441638:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941513.441639:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941513.441641:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941513.441644:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90262000. 00000100:00000001:3.0:1773941513.441647:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941513.441651:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90275500 x1860112241163520/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941513.441657:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941513.441659:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941513.441662:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007829602] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90275500 x1860112241163520/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941513.441677:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941513.441679:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941513.441680:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941513.441682:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941513.441684:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941513.441685:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941513.441686:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941513.441688:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941513.441690:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941513.441695:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241163392/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941529 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941513.441730:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941513.441733:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941513.441734:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941513.441737:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1773941513.441741:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941513.441746:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275500 x1860112241163520/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941513.441747:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941513.441751:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941513.441757:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941513.441758:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941513.441761:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941513.441764:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941513.441766:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941513.441770:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241163392/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941529 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941513.441771:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241163520:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941513.441779:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941513.441782:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941513.441784:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941513.441786:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1773941513.441788:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241163392:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000010:0.0:1773941513.441791:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0000. 00000100:00000001:3.0:1773941513.441794:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1773941513.441795:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941513.441800:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:0.0:1773941513.441804:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941513.441805:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941513.441808:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000010:0.0:1773941513.441808:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000100:00000040:3.0:1773941513.441814:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241163392/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941529 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941513.441814:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941513.441820:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241163520, portal 25 00000100:00000001:3.0:1773941513.441824:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941513.441825:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941513.441828:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109622528 : -128249599929088 : ffff8b5b90275500) 00000100:00000010:3.0:1773941513.441829:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941513.441832:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941513.441833:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:3.0:1773941513.441838:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:3.0:1773941513.441841:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941513.441844:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:0.0:1773941513.441844:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275500 x1860112241163520/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941523 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1773941513.441845:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 00000020:00000001:3.0:1773941513.441850:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941513.441855:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000100:00000001:0.0:1773941513.441861:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:3.0:1773941513.441862:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1def lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941513.441866:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941513.441870:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6cd48. 00010000:00000001:3.0:1773941513.441876:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000200:0.0:1773941513.441877:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241163520, offset 0 00010000:00010000:3.0:1773941513.441881:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1def lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941513.441891:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:0.0:1773941513.441891:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:3.0:1773941513.441895:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1def lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941513.441899:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941513.441905:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941513.441908:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941513.441911:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941513.441915:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941513.441919:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941513.441924:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[150, 151] 00010000:00000001:3.0:1773941513.441925:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941513.441929:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:0.0:1773941513.441929:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00010000:3.0:1773941513.441933:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1def lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941513.441934:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [257:257:256:4294967295] 192.168.204.143@tcp LPNI seq info [152:152:8:4294967295] 00000400:00000200:0.0:1773941513.441945:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941513.441949:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941513.441953:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941513.441956:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:3.0:1773941513.441958:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:3.0:1773941513.441962:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1def lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941513.441962:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941513.441970:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00010000:00010000:3.0:1773941513.441971:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:3.0:1773941513.441974:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:0.0:1773941513.441977:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00010000:3.0:1773941513.441978:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1def lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941513.441986:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:3.0:1773941513.441987:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:3.0:1773941513.441991:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000800:00000200:0.0:1773941513.441991:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00000020:00000001:3.0:1773941513.441994:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941513.441996:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000020:00000001:3.0:1773941513.441998:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941513.441999:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941513.442001:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:3.0:1773941513.442003:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1def lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941513.442004:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941513.442011:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941513.442015:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294953805 10000000:00000001:3.0:1773941513.442016:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941513.442019:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1def lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941513.442020:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294953803 00000100:00000001:0.0:1773941513.442026:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294953805 10000000:01000000:3.0:1773941513.442027:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:0.0:1773941513.442030:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294953803 10000000:00000040:3.0:1773941513.442035:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941513.442036:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941513.442039:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941513.442043:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:0.0:1773941513.442044:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1773941513.442046:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:0.0:1773941513.442047:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941513.442049:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:0.0:1773941513.442051:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941513.442053:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:0.0:1773941513.442054:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1773941513.442056:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941513.442059:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1def from hash 00010000:00000001:3.0:1773941513.442061:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:0.0:1773941513.442061:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:3.0:1773941513.442062:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941513.442064:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:3.0:1773941513.442065:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941513.442069:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:0.0:1773941513.442070:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:3.0:1773941513.442071:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:0.0:1773941513.442072:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:3.0:1773941513.442074:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:0.0:1773941513.442075:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941513.442079:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941513.442079:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941513.442083:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:2.0:1773941513.442083:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:3.0:1773941513.442086:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941513.442089:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1773941513.442089:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00010000:00010000:3.0:1773941513.442095:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1def lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941513.442098:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:3.0:1773941513.442103:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941513.442104:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941513.442105:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941513.442106:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941513.442107:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941513.442108:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941513.442108:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941513.442110:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1def lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:3.0:1773941513.442116:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962800 count: 0 00000400:00000200:2.0:1773941513.442121:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00010000:00000001:3.0:1773941513.442126:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941513.442127:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6cd48 00000400:00000010:2.0:1773941513.442131:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6cd48. 00000100:00000001:2.0:1773941513.442134:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941513.442139:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275500 x1860112241163520/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941523 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941513.442148:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941513.442152:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275500 x1860112241163520/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941523 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941513.442160:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941513.442162:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1773941513.442175:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941513.442178:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941513.442188:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241163392/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941529 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941513.442194:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941513.442195:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941513.442197:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941513.442199:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941513.442201:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941513.442202:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941513.442204:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941513.442206:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2600. 02000000:00000001:3.0:1773941513.442208:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941513.442209:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941513.442210:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:3.0:1773941513.442212:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941513.442213:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941513.442214:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941513.442216:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941513.442218:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941513.442221:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068e00. 00000020:00000010:3.0:1773941513.442223:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b839e2600. 00000020:00000010:3.0:1773941513.442227:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b839e2a00. 00000040:00000010:3.0:1773941513.442230:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b839e3000. 00000020:00000010:3.0:1773941513.442233:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941513.442236:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b839e3a00. 80000000:00000010:3.0:1773941513.442240:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941513.442244:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b84d8ae00. 20000000:00000010:3.0:1773941513.442248:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b84d8a200. 10000000:00000040:3.0:1773941513.442252:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:3.0:1773941513.442253:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941513.442256:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b839e2a00. 00000040:00000010:3.0:1773941513.442259:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b839e3000. 00000020:00000010:3.0:1773941513.442261:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941513.442264:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b839e3a00. 80000000:00000010:3.0:1773941513.442267:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941513.442274:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b84d8ae00. 20000000:00000010:3.0:1773941513.442277:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b84d8a200. 00000020:00000010:3.0:1773941513.442280:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b839e2600. 10000000:00000010:3.0:1773941513.442283:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068e00. 10000000:01000000:3.0:1773941513.442286:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941513.442289:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941513.442291:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941513.442293:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941513.442294:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941513.442296:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941513.442299:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941513.443441:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941513.443450:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941513.443453:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941513.443455:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941513.443460:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941513.443465:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941513.443471:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941513.443475:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941513.443478:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941513.443489:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607d00 00000400:00000200:2.0:1773941513.443494:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941513.443499:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x122d [1] + 0 00000800:00000001:2.0:1773941513.443505:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941513.443532:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941513.443535:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941513.443540:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941513.443544:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941513.443549:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275500 x1860112241163520/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941523 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941513.443561:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90275500 x1860112241163520/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941523 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941513.443579:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941513.443585:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941513.443589:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941513.443604:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941513.443607:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941513.443611:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941513.443612:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941513.443614:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941513.443617:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941513.443619:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941513.443620:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941513.443625:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275500 x1860112241163520/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941523 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941513.443633:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275500 x1860112241163520/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941523 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941513.443637:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941513.443638:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941513.443640:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941513.443641:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941513.443643:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941513.443644:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941513.443646:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941513.443649:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=92 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941513.443651:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941513.443653:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941513.443654:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941513.443655:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941513.443657:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941513.443658:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941513.443667:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275500 x1860112241163520/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941523 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941513.443672:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941513.443674:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941513.443675:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941513.443677:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941513.443680:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941513.443682:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941513.443683:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941513.443684:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941513.443686:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941513.443687:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941513.443689:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941513.443692:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941513 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941513.443693:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941513.443699:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941513.443700:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941513.443725:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941513.443727:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941513.443729:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941513.443732:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941513.443734:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941513.443736:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941513.443739:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941513.443742:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941513.443744:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941513.443930:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941513.443934:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941513.443941:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275500 x1860112241163520/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941523 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941513.443951:0:12235:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:1.0:1773941513.443955:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241163520:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941513.443961:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941513.443961:0:12235:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:1.0:1773941513.443966:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:0.0:1773941513.443967:0:12235:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941513.443971:0:12235:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000040:1.0:1773941513.443972:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275500 x1860112241163520/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941523 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:01000000:0.0:1773941513.443973:0:12235:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941513.443977:0:12235:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:1.0:1773941513.443981:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941513.443985:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941513.443986:0:12235:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:1.0:1773941513.443989:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0000. 00000001:00000001:0.0:1773941513.443990:0:12235:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941513.443993:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941513.443994:0:12235:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:1.0:1773941513.443996:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941513.443997:0:12235:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:1.0:1773941513.444000:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941513.444001:0:12235:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941513.444004:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000001:0.0:1773941513.444005:0:12235:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941513.444008:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90262000. 00000100:00000001:0.0:1773941513.444009:0:12235:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941513.444015:0:12235:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:1.0:1773941513.444016:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1773941513.444018:0:12235:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941513.444020:0:12235:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941513.444021:0:12235:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941513.444022:0:12235:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941513.444024:0:12235:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:1.0:1773941513.444028:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:01000000:0.0:1773941513.444028:0:12235:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941513.444032:0:12235:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000010:1.0:1773941513.444033:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275500. 00000100:00000001:1.0:1773941513.444039:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941513.444041:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941513.444043:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941513.444045:0:12235:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941513.444047:0:12235:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941513.444049:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941513.444049:0:12235:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941513.444054:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:02000000:0.0:1773941513.444054:0:12235:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941513.444061:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941513.444062:0:12235:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941513.444065:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941513.444066:0:12235:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941513.444068:0:12235:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941513.444072:0:12235:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941513.444074:0:12235:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941513.447746:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962800. 00010000:00000010:3.1:1773941513.447757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:0.0:1773941514.079854:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941514.079872:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941514.079878:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941514.143818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1773941514.143827:0:36:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1773941514.143837:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b850aad80. 00000020:00000040:0.0:1773941514.143842:0:36:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941514.143846:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941514.143849:0:36:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941514.143854:0:36:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941514.143863:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b99101000. 00000100:00000001:0.0:1773941514.143870:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941514.143897:0:36:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS req@ffff8b5b850aad80 x1860112241163648/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941514.143909:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941514.143914:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941514.143920:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e97c40a7] to pc [ptlrpcd_00_01+1] req@ffff8b5b850aad80 x1860112241163648/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941514.143945:0:36:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941514.143949:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (289) 00000100:00000001:0.0:1773941514.143959:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1773941514.143963:0:2352:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:0.0:1773941514.143965:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941514.143971:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941514.143973:0:2352:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941514.143976:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.143978:0:2352:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941514.143984:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941514.143986:0:2352:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941514.143988:0:2352:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941514.143994:0:2352:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b850aad80 x1860112241163648/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941514.144003:0:2352:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941514.144006:0:2352:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941514.144010:0:2352:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941514.144012:0:2352:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941514.144015:0:2352:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941514.144019:0:2352:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1080e6b5-6a6c-4762-b0ec-c785990fec17:2352:1860112241163648:192.168.204.143@tcp:400:kworker.0 00000100:00000001:0.0:1773941514.144024:0:2352:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941514.144029:0:2352:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941514.144032:0:2352:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941514.144034:0:2352:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941514.144039:0:2352:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b845c4a00. 02000000:00000001:0.0:1773941514.144042:0:2352:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941514.144049:0:2352:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941514.144053:0:2352:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941514.144061:0:2352:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941514.144068:0:2352:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241163648, portal 25 00000100:00000001:0.0:1773941514.144077:0:2352:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941514.144079:0:2352:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615823923195264 : -128249786356352 : ffff8b5b850aad80) 00000100:00000040:0.0:1773941514.144086:0:2352:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b850aad80 x1860112241163648/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941514.144094:0:2352:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941514.144098:0:2352:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941514.144101:0:2352:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6cbb0. 00000100:00000200:0.0:1773941514.144106:0:2352:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241163648, offset 0 00000400:00000200:0.0:1773941514.144112:0:2352:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941514.144121:0:2352:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941514.144128:0:2352:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941514.144133:0:2352:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941514.144137:0:2352:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941514.144142:0:2352:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941514.144145:0:2352:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941514.144149:0:2352:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[152, 151] 00000400:00000200:0.0:1773941514.144153:0:2352:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:0.0:1773941514.144157:0:2352:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [258:258:256:4294967295] 192.168.204.153@tcp LPNI seq info [153:153:8:4294967295] 00000400:00000200:0.0:1773941514.144162:0:2352:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:0.0:1773941514.144172:0:2352:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:0.0:1773941514.144179:0:2352:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:0.0:1773941514.144185:0:2352:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b91e1e300. 00000800:00000200:0.0:1773941514.144192:0:2352:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:0.0:1773941514.144199:0:2352:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:0.0:1773941514.144203:0:2352:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b91e1e300 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1773941514.144215:0:2352:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941514.144218:0:2352:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941514.144220:0:2352:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294954507 00000100:00000001:0.0:1773941514.144226:0:2352:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294953805 00000100:00000001:0.0:1773941514.144230:0:2352:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294954507 00000100:00000001:0.0:1773941514.144240:0:2352:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294953805 00000100:00000001:0.0:1773941514.144242:0:2352:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941514.144244:0:2352:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.144247:0:2352:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.144249:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941514.144323:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941514.144329:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b91e1e300. 00000400:00000200:2.0:1773941514.144340:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941514.144346:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941514.144352:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6cbb0 00000400:00000010:2.0:1773941514.144353:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6cbb0. 00000100:00000001:2.0:1773941514.144357:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941514.144361:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b850aad80 x1860112241163648/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.144369:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941514.144372:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b850aad80 x1860112241163648/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941530 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.144377:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941514.144379:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941514.145398:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941514.145413:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941514.145416:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941514.145429:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941514.145437:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941514.145443:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941514.145451:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941514.145455:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941514.145460:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941514.145464:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607d80 00000400:00000200:2.0:1773941514.145468:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941514.145473:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x1235 [1] + 224 00000800:00000001:2.0:1773941514.145481:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941514.145488:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941514.145491:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941514.145494:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941514.145511:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941514.145518:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b850aad80 x1860112241163648/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941530 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941514.145527:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b850aad80 x1860112241163648/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941530 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.145548:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941514.145558:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941514.145562:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941514.145595:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941514.145599:0:2352:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941514.145605:0:2352:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941514.145607:0:2352:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941514.145610:0:2352:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941514.145615:0:2352:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941514.145616:0:2352:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941514.145619:0:2352:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941514.145624:0:2352:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b850aad80 x1860112241163648/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941530 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941514.145633:0:2352:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b850aad80 x1860112241163648/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941530 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941514.145638:0:2352:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941514.145640:0:2352:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941514.145642:0:2352:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941514.145645:0:2352:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941514.145647:0:2352:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941514.145650:0:2352:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941514.145652:0:2352:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941514.145658:0:2352:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=93 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941514.145661:0:2352:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941514.145664:0:2352:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941514.145666:0:2352:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941514.145669:0:2352:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941514.145671:0:2352:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941514.145676:0:2352:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941514.145680:0:2352:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941514.145683:0:2352:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941514.145692:0:2352:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941514.145694:0:2352:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941514.145696:0:2352:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941514.145700:0:2352:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b850aad80 x1860112241163648/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941530 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941514.145731:0:2352:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941514.145732:0:2352:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941514.145734:0:2352:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941514.145738:0:2352:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941514.145740:0:2352:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941514.145742:0:2352:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.145743:0:2352:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941514.145744:0:2352:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941514.145745:0:2352:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941514.145746:0:2352:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.145747:0:2352:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941514.145749:0:2352:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941514.145750:0:2352:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941514.145753:0:2352:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941514.145755:0:2352:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 283 00000400:00000200:0.0:1773941514.145759:0:2352:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941514.145763:0:2352:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941514.145766:0:2352:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941514.145769:0:2352:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941514.145771:0:2352:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941514.145773:0:2352:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:0.0:1773941514.145776:0:2352:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941514.145778:0:2352:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941514.145780:0:2352:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941514.145783:0:2352:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941514.145784:0:2352:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941514.145786:0:2352:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:0.0:1773941514.145788:0:2352:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941514.145802:0:2352:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941514.145804:0:2352:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941514.145806:0:2352:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:0.0:1773941514.145808:0:2352:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:0.0:1773941514.145811:0:2352:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:0.0:1773941514.145813:0:2352:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941514.145815:0:2352:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941514.145818:0:2352:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941514.145820:0:2352:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.145822:0:2352:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941514.145823:0:2352:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941514.145827:0:2352:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b850ab800. 00000020:00000040:0.0:1773941514.145830:0:2352:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:0.0:1773941514.145834:0:2352:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941514.145836:0:2352:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.145839:0:2352:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941514.145842:0:2352:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941514.145845:0:2352:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941514.145852:0:2352:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b850c0000. 00000100:00000001:0.0:1773941514.145863:0:2352:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941514.145871:0:2352:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b850ab800 x1860112241163776/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941514.145880:0:2352:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941514.145884:0:2352:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941514.145889:0:2352:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043b1e523] to pc [ptlrpcd_rcv+-1] req@ffff8b5b850ab800 x1860112241163776/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941514.145912:0:2352:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.145916:0:2352:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941514.145918:0:2352:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:0.0:1773941514.145924:0:2352:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b850aad80 x1860112241163648/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941530 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941514.145933:0:2352:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941514.145944:0:2352:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941514.145947:0:2352:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941514.145952:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1773941514.145954:0:2352:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b850aad80 x1860112241163648/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941530 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941514.145957:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941514.145962:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:0.0:1773941514.145977:0:2352:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:1080e6b5-6a6c-4762-b0ec-c785990fec17:2352:1860112241163648:192.168.204.143@tcp:400:kworker.0 00000100:00000040:1.0:1773941514.145979:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b850ab800 x1860112241163776/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941514.145993:0:2352:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941514.146004:0:2352:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941514.146005:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941514.146011:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941514.146018:0:2352:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b850aad80 x1860112241163648/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941530 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941514.146019:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941514.146024:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941514.146031:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941514.146045:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241163776:192.168.204.143@tcp:250:ptlrpcd_00_00.0 00000100:00000001:0.0:1773941514.146046:0:2352:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941514.146050:0:2352:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941514.146057:0:2352:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b845c4a00. 00000100:00000001:1.0:1773941514.146058:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941514.146067:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941514.146067:0:2352:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941514.146071:0:2352:0:(genops.c:1236:class_import_put()) Process entered 02000000:00000001:1.0:1773941514.146072:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941514.146073:0:2352:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941514.146079:0:2352:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:1.0:1773941514.146081:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941514.146083:0:2352:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b99101000. 02000000:00000001:0.0:1773941514.146090:0:2352:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000010:1.0:1773941514.146091:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735400. 02000000:00000001:0.0:1773941514.146092:0:2352:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941514.146095:0:2352:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b850aad80. 02000000:00000001:1.0:1773941514.146099:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.146101:0:2352:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941514.146107:0:2352:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941514.146110:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941514.146116:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941514.146118:0:2352:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941514.146121:0:2352:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1773941514.146123:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000100:00000001:0.0:1773941514.146124:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941514.146128:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1773941514.146133:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941514.146141:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241163776, portal 25 00000100:00000001:1.0:1773941514.146144:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941514.146145:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615823923197952 : -128249786353664 : ffff8b5b850ab800) 00000100:00000040:1.0:1773941514.146153:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b850ab800 x1860112241163776/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941514.146170:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941514.146173:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941514.146175:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1770. 00000100:00000200:1.0:1773941514.146178:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241163776, offset 0 00000400:00000200:1.0:1773941514.146183:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941514.146189:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941514.146194:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941514.146197:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941514.146200:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941514.146202:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941514.146205:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941514.146208:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[152, 153] 00000400:00000200:1.0:1773941514.146213:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941514.146216:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [259:259:256:4294967295] 192.168.204.143@tcp LPNI seq info [154:154:8:4294967295] 00000400:00000200:1.0:1773941514.146221:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941514.146229:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941514.146234:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941514.146239:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024be00. 00000800:00000200:1.0:1773941514.146244:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941514.146251:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941514.146254:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024be00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:1.0:1773941514.146269:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941514.146274:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941514.146277:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294954509 00000100:00000001:1.0:1773941514.146284:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294954507 00000100:00000001:1.0:1773941514.146290:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294954509 00000100:00000001:1.0:1773941514.146295:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294954507 00000100:00000001:1.0:1773941514.146300:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941514.146302:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941514.146312:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941514.146315:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941514.146321:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941514.146323:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1773941514.146325:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941514.146327:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941514.146329:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941514.146332:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941514.146343:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941514.146347:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024be00. 00000400:00000200:2.0:1773941514.146352:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941514.146356:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941514.146359:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1770 00000400:00000010:2.0:1773941514.146361:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1770. 00000100:00000001:2.0:1773941514.146364:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941514.146368:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b850ab800 x1860112241163776/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.146374:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941514.146376:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b850ab800 x1860112241163776/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.146387:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941514.146388:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941514.147937:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941514.147943:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941514.147945:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941514.147946:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941514.147950:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941514.147954:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941514.147958:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941514.147961:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941514.147963:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941514.147966:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607e00 00000400:00000200:2.0:1773941514.147969:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941514.147972:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x123d [1] + 0 00000800:00000001:2.0:1773941514.147976:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941514.147981:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941514.147983:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941514.147987:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941514.147990:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941514.147993:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b850ab800 x1860112241163776/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941514.148000:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b850ab800 x1860112241163776/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.148013:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941514.148017:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941514.148019:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941514.148058:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941514.148062:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941514.148067:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941514.148069:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941514.148072:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941514.148078:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941514.148080:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941514.148083:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941514.148090:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b850ab800 x1860112241163776/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941514.148103:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b850ab800 x1860112241163776/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941514.148112:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941514.148113:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941514.148116:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941514.148118:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941514.148120:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941514.148123:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941514.148126:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941514.148131:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=93 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941514.148135:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941514.148138:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941514.148140:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941514.148142:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941514.148144:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941514.148148:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941514.148153:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b850ab800 x1860112241163776/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941524 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941514.148175:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941514.148178:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941514.148180:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941514.148183:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1773941514.148190:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:1.0:1773941514.148193:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941514.148195:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941514.148197:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941514.148201:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941514.148203:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941514.148206:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:1.0:1773941514.148210:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941514 v=0 (0 0 0 0) 00000100:00001000:1.0:1773941514.148214:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:1.0:1773941514.148220:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:1.0:1773941514.148223:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941514.148226:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:1.0:1773941514.148229:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941514.148230:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941514.148234:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:1.0:1773941514.148237:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:1.0:1773941514.148239:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:1.0:1773941514.148243:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:1.0:1773941514.148247:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:1.0:1773941514.148249:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941514.148547:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.148552:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941514.148559:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b850ab800 x1860112241163776/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941524 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941514.148567:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241163776:192.168.204.143@tcp:250:ptlrpcd_00_00.0 00000100:00000001:0.0:1773941514.148579:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941514.148581:0:12236:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:0.0:1773941514.148582:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941514.148589:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b850ab800 x1860112241163776/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941524 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941514.148595:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00080000:1.0:1773941514.148601:0:12236:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 02000000:00000001:0.0:1773941514.148601:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941514.148603:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735400. 00000100:00100000:1.0:1773941514.148609:0:12236:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:0.0:1773941514.148610:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941514.148614:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941514.148617:0:12236:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000040:0.0:1773941514.148617:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941514.148623:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 10000000:01000000:1.0:1773941514.148624:0:12236:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000010:0.0:1773941514.148626:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b850c0000. 02000000:00000001:0.0:1773941514.148631:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:1.0:1773941514.148634:0:12236:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:0.0:1773941514.148634:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941514.148641:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b850ab800. 00000001:00000001:1.0:1773941514.148644:0:12236:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:0.0:1773941514.148646:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941514.148647:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941514.148649:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941514.148652:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941514.148657:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.148659:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000001:00000001:1.0:1773941514.148660:0:12236:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.148661:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941514.148667:0:12236:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941514.148670:0:12236:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941514.148673:0:12236:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941514.148676:0:12236:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941514.148679:0:12236:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941514.148683:0:12236:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941514.148686:0:12236:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941514.148688:0:12236:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941514.148690:0:12236:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941514.148693:0:12236:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941514.148696:0:12236:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941514.148697:0:12236:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941514.148699:0:12236:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941514.148749:0:12236:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941514.148752:0:12236:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941514.148755:0:12236:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941514.148762:0:12236:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941514.148770:0:12236:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941514.148775:0:12236:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941514.148777:0:12236:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941514.148780:0:12236:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941514.148782:0:12236:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941514.911837:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941514.911848:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941514.911855:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941514.911858:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941514.911860:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941514.911862:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941514.911865:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941514.911868:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941514.911871:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941514.911875:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941514.911878:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941514.911895:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941514.911901:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941514.911906:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941514.911909:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941514.911916:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941514.911934:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b84d8a200. 00000100:00000001:3.0:1773941514.911943:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941514.911949:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941514.911951:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941514.911959:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962800. 00010000:00000001:3.0:1773941514.911965:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941514.911968:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941514.911973:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941514.911975:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1df6 to hash 00000020:00000001:3.0:1773941514.911999:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941514.912000:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941514.912003:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941514.912007:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941514.912010:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941514.912014:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1df6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941514.912023:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1df6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941514.912035:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1df6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941514.912040:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941514.912042:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941514.912048:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941514.912050:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941514.912052:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941514.912054:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941514.912059:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941514.912063:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941514.912065:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941514.912067:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941514.912078:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241163904/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941514.912085:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941514.912087:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941514.912091:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941514.912093:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941514.912095:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941514.912098:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241163904:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941514.912109:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941514.912115:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941514.912118:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941514.912120:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941514.912124:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941514.912126:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941514.912134:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941514.912138:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941514.912144:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941514.912152:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241163904, portal 25 00000100:00000001:3.0:1773941514.912154:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941514.912155:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941514.912159:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241163904/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941530 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941514.912165:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941514.912167:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941514.912169:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a198. 00000100:00000200:3.0:1773941514.912172:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241163904, offset 0 00000400:00000200:3.0:1773941514.912177:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941514.912186:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941514.912194:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941514.912199:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941514.912204:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941514.912207:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941514.912210:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941514.912212:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[154, 153] 00000400:00000200:3.0:1773941514.912215:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941514.912218:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [260:260:256:4294967295] 192.168.204.153@tcp LPNI seq info [155:155:8:4294967295] 00000400:00000200:3.0:1773941514.912222:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941514.912230:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941514.912235:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941514.912243:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69500. 00000800:00000200:3.0:1773941514.912250:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941514.912257:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941514.912262:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69500 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941514.912281:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941514.912285:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941514.912288:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294955275 00000100:00000001:3.0:1773941514.912295:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294954509 00000100:00000001:3.0:1773941514.912301:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294955275 00000100:00000001:3.0:1773941514.912305:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294954509 00000100:00000001:3.0:1773941514.912316:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941514.912319:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941514.912325:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941514.912327:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941514.912331:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941514.912334:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941514.912338:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941514.912466:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941514.912472:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69500. 00000400:00000200:2.0:1773941514.912480:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941514.912501:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941514.912507:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a198 00000400:00000010:2.0:1773941514.912525:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a198. 00000100:00000001:2.0:1773941514.912532:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941514.912546:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241163904/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941530 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.912559:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941514.912562:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241163904/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941530 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.912570:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941514.912572:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941514.913574:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941514.913588:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941514.913592:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941514.913595:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941514.913600:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941514.913606:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941514.913614:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941514.913619:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941514.913625:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941514.913630:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607e80 00000400:00000200:2.0:1773941514.913635:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941514.913641:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x1245 [1] + 224 00000800:00000001:2.0:1773941514.913650:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941514.913659:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941514.913661:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941514.913663:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941514.913667:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941514.913670:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241163904/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941530 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941514.913678:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241163904/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941530 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.913692:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941514.913723:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941514.913731:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941514.913732:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941514.913735:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941514.913739:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941514.913740:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000800:00000001:2.0:1773941514.913742:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000100:00000001:3.0:1773941514.913743:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000001:2.0:1773941514.913747:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1773941514.913751:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241163904/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941530 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941514.913761:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241163904/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941530 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941514.913766:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941514.913767:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941514.913770:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941514.913773:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941514.913775:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941514.913783:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941514.913785:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941514.913791:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=93 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941514.913794:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941514.913797:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941514.913801:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941514.913804:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941514.913808:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941514.913812:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941514.913817:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941514.913822:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941514.913826:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941514.913829:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941514.913831:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941514.913833:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941514.913838:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241163904/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941530 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941514.913847:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941514.913849:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941514.913851:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941514.913855:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941514.913858:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941514.913860:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941514.913862:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941514.913864:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941514.913865:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941514.913867:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941514.913869:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941514.913872:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941514.913874:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941514.913878:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941514.913881:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 284 00000400:00000200:3.0:1773941514.913886:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941514.913895:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941514.913899:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941514.913902:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941514.913904:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941514.913906:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941514.913909:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941514.913910:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941514.913912:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941514.913914:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941514.913916:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941514.913917:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941514.913919:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941514.913920:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941514.913922:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941514.913923:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941514.913934:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941514.913936:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941514.913938:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941514.913940:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941514.913943:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941514.913945:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941514.913947:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941514.913948:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941514.913951:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275880. 00000020:00000040:3.0:1773941514.913953:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941514.913956:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941514.913957:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941514.913959:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941514.913961:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941514.913963:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941514.913966:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90262000. 00000100:00000001:3.0:1773941514.913968:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941514.913973:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90275880 x1860112241164032/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941514.913984:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941514.913986:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941514.913989:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000766119e1] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90275880 x1860112241164032/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941514.914007:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941514.914009:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941514.914010:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941514.914012:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941514.914014:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941514.914015:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941514.914017:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941514.914018:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941514.914020:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941514.914024:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241163904/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941530 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941514.914029:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941514.914032:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941514.914034:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941514.914036:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941514.914039:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241163904/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941530 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941514.914041:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941514.914046:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1773941514.914048:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241163904:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941514.914049:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1773941514.914056:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941514.914058:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:0.0:1773941514.914059:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275880 x1860112241164032/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941514.914060:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941514.914063:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941514.914070:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241163904/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941530 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941514.914070:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941514.914073:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941514.914078:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941514.914081:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941514.914082:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941514.914085:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941514.914089:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241164032:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000010:3.0:1773941514.914091:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:0.0:1773941514.914096:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1773941514.914097:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941514.914099:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:3.0:1773941514.914102:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:3.0:1773941514.914104:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:0.0:1773941514.914106:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1773941514.914107:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:0.0:1773941514.914109:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941514.914110:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 02000000:00000001:0.0:1773941514.914113:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1773941514.914114:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941514.914118:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 02000000:00000010:0.0:1773941514.914118:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0000. 00010000:00010000:3.0:1773941514.914123:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1df6 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941514.914123:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941514.914128:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941514.914132:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941514.914136:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941514.914141:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241164032, portal 25 00010000:00000001:3.0:1773941514.914143:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941514.914144:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00010000:3.0:1773941514.914149:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1df6 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941514.914149:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109623424 : -128249599928192 : ffff8b5b90275880) 00010000:00000001:3.0:1773941514.914158:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000040:0.0:1773941514.914159:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275880 x1860112241164032/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1773941514.914161:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1df6 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941514.914178:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941514.914180:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941514.914181:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:3.0:1773941514.914183:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1df6 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941514.914187:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941514.914192:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c4c8. 00010000:00000001:3.0:1773941514.914193:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941514.914196:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1773941514.914196:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241164032, offset 0 00010000:00000001:3.0:1773941514.914199:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:3.0:1773941514.914203:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1df6 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941514.914203:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941514.914210:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:3.0:1773941514.914213:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:0.0:1773941514.914215:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941514.914219:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941514.914223:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 10000000:00000001:3.0:1773941514.914225:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941514.914227:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00010000:3.0:1773941514.914231:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1df6 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941514.914231:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941514.914236:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[154, 155] 00000400:00000200:0.0:1773941514.914245:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941514.914250:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [261:261:256:4294967295] 192.168.204.143@tcp LPNI seq info [156:156:8:4294967295] 00010000:00000001:3.0:1773941514.914254:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:0.0:1773941514.914254:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941514.914259:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941514.914263:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1773941514.914265:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000020:00000040:3.0:1773941514.914267:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000800:00000200:0.0:1773941514.914271:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000020:00000001:3.0:1773941514.914273:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000800:00000010:0.0:1773941514.914276:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b91e1e300. 00010000:00000001:3.0:1773941514.914278:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000800:00000200:0.0:1773941514.914283:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:3.0:1773941514.914285:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:3.0:1773941514.914290:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1df6 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941514.914298:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 10000000:00000001:3.0:1773941514.914303:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:0.0:1773941514.914303:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b91e1e300 type 1, nob 616 niov 1 nkiov 1 10000000:00010000:3.0:1773941514.914308:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1df6 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1773941514.914318:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:0.0:1773941514.914318:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:3.0:1773941514.914321:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941514.914325:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941514.914325:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941514.914328:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294955277 00010000:00010000:3.0:1773941514.914331:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941514.914334:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:0.0:1773941514.914334:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294955275 00010000:00000001:3.0:1773941514.914338:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:0.0:1773941514.914339:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294955277 00010000:00000001:3.0:1773941514.914340:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941514.914344:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:0.0:1773941514.914345:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294955275 00010000:00000001:3.0:1773941514.914346:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.914349:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1773941514.914350:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1df6 from hash 00000100:00000001:0.0:1773941514.914352:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941514.914353:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:0.0:1773941514.914356:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941514.914357:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941514.914360:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941514.914362:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:0.0:1773941514.914362:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1773941514.914365:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941514.914367:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:3.0:1773941514.914369:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941514.914370:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941514.914372:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1773941514.914374:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941514.914375:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941514.914378:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941514.914379:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941514.914380:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941514.914382:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941514.914385:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941514.914388:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1df6 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941514.914392:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941514.914393:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941514.914394:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941514.914395:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941514.914396:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941514.914397:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941514.914398:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941514.914400:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1df6 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:3.0:1773941514.914405:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962800 count: 0 00010000:00000001:3.0:1773941514.914416:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941514.914417:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941514.914420:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941514.914423:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241163904/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941530 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941514.914428:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941514.914431:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941514.914433:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941514.914436:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941514.914437:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941514.914439:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941514.914441:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941514.914444:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b84d8a200. 02000000:00000001:3.0:1773941514.914447:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941514.914449:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941514.914451:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:3.0:1773941514.914453:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941514.914455:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941514.914458:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000200:2.0:1773941514.914459:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941514.914464:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b91e1e300. 10000000:01000000:3.0:1773941514.914469:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000400:00000200:2.0:1773941514.914470:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000001:3.0:1773941514.914476:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000400:00000200:2.0:1773941514.914478:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941514.914482:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c4c8 10000000:00000010:3.0:1773941514.914483:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068880. 00000400:00000010:2.0:1773941514.914498:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c4c8. 00000100:00000001:2.0:1773941514.914503:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941514.914508:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275880 x1860112241164032/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.914515:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941514.914517:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275880 x1860112241164032/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.914523:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941514.914524:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:3.0:1773941514.914628:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b84d8a200. 00000020:00000010:3.0:1773941514.914633:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b84d8ae00. 00000040:00000010:3.0:1773941514.914637:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b84d8bc00. 00000020:00000010:3.0:1773941514.914641:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941514.914646:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b839e2600. 80000000:00000010:3.0:1773941514.914650:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941514.914654:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b839e3a00. 20000000:00000010:3.0:1773941514.914658:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b839e3000. 10000000:00000040:3.0:1773941514.914663:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:3.0:1773941514.914664:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941514.914669:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b84d8ae00. 00000040:00000010:3.0:1773941514.914672:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b84d8bc00. 00000020:00000010:3.0:1773941514.914676:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941514.914679:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b839e2600. 80000000:00000010:3.0:1773941514.914682:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941514.914686:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b839e3a00. 20000000:00000010:3.0:1773941514.914688:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b839e3000. 00000020:00000010:3.0:1773941514.914690:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84d8a200. 10000000:00000010:3.0:1773941514.914692:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068880. 10000000:01000000:3.0:1773941514.914695:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941514.914697:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941514.914699:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941514.914733:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941514.914736:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941514.914737:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941514.914740:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941514.915810:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941514.915819:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941514.915821:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941514.915824:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941514.915829:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941514.915834:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941514.915840:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941514.915844:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941514.915853:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941514.915857:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321607f00 00000400:00000200:2.0:1773941514.915862:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941514.915866:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1255 [1] + 0 00000800:00000001:2.0:1773941514.915872:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941514.915880:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941514.915882:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941514.915886:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941514.915890:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941514.915902:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275880 x1860112241164032/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941514.915913:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90275880 x1860112241164032/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.915933:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941514.915940:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941514.915944:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941514.915968:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941514.915971:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941514.915974:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941514.915976:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941514.915978:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941514.915981:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941514.915983:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941514.915985:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941514.915990:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275880 x1860112241164032/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941514.915998:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275880 x1860112241164032/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941514.916003:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941514.916004:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941514.916006:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941514.916013:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941514.916014:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941514.916015:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941514.916017:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941514.916020:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=93 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941514.916023:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941514.916025:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941514.916026:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941514.916028:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941514.916030:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.916032:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941514.916036:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275880 x1860112241164032/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941524 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941514.916041:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941514.916043:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941514.916044:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941514.916046:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941514.916050:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941514.916052:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941514.916053:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941514.916054:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941514.916055:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941514.916057:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941514.916059:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941514.916063:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941514 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941514.916065:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941514.916071:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941514.916072:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941514.916073:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941514.916075:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.916076:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941514.916079:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941514.916081:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941514.916082:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941514.916089:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941514.916092:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941514.916094:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941514.916356:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941514.916361:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941514.916367:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275880 x1860112241164032/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941524 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941514.916375:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241164032:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941514.916378:0:12237:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:1.0:1773941514.916380:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941514.916383:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941514.916387:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275880 x1860112241164032/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941524 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941514.916389:0:12237:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941514.916395:0:12237:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941514.916399:0:12237:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941514.916401:0:12237:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941514.916405:0:12237:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:1.0:1773941514.916406:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941514.916408:0:12237:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941514.916412:0:12237:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941514.916413:0:12237:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941514.916414:0:12237:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:1.0:1773941514.916418:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941514.916418:0:12237:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941514.916422:0:12237:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941514.916424:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0000. 00000100:00000001:0.0:1773941514.916424:0:12237:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941514.916428:0:12237:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:1.0:1773941514.916431:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1773941514.916432:0:12237:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000001:1.0:1773941514.916436:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941514.916436:0:12237:0:(recover.c:147:ptlrpc_resend()) Process entered 00000020:00000040:1.0:1773941514.916440:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941514.916441:0:12237:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941514.916444:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:0.0:1773941514.916446:0:12237:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 02000000:00000010:1.0:1773941514.916448:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90262000. 00000100:00000001:0.0:1773941514.916449:0:12237:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:1.0:1773941514.916453:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:01000000:0.0:1773941514.916453:0:12237:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 02000000:00000001:1.0:1773941514.916457:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:00000040:0.0:1773941514.916457:0:12237:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000010:1.0:1773941514.916459:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275880. 00000100:00000001:1.0:1773941514.916463:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941514.916464:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941514.916465:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941514.916469:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941514.916470:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941514.916472:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941514.916473:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941514.916474:0:12237:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941514.916477:0:12237:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941514.916478:0:12237:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941514.916481:0:12237:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941514.916499:0:12237:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941514.916503:0:12237:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941514.916505:0:12237:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941514.916507:0:12237:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941514.916509:0:12237:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941514.919750:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962800. 00010000:00000010:3.1:1773941514.919759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941515.103867:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941515.103877:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941515.103884:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1773941516.127867:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941516.127874:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941516.127897:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941516.191796:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941516.191802:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941516.191822:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941516.191825:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941516.191826:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941516.191827:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941516.191830:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941516.191831:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941516.191833:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941516.191836:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941516.191837:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941516.191845:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941516.191849:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941516.191854:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941516.191857:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941516.191862:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941516.191869:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2600. 00000100:00000001:3.0:1773941516.191876:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941516.191879:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941516.191882:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941516.191889:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962900. 00010000:00000001:3.0:1773941516.191894:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941516.191897:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941516.191902:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941516.191904:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1dfd to hash 00000020:00000001:3.0:1773941516.191906:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941516.191907:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941516.191910:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941516.191914:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941516.191916:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941516.191919:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1dfd lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941516.191928:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1dfd lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941516.191938:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1dfd lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941516.191949:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941516.191951:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941516.191957:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941516.191959:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941516.191962:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941516.191964:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941516.191968:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941516.191972:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941516.191974:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941516.191976:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941516.191985:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241164160/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.191992:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941516.191995:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941516.191998:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941516.192000:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941516.192002:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941516.192005:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241164160:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941516.192008:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941516.192012:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941516.192014:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941516.192016:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941516.192020:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941516.192022:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941516.192031:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941516.192036:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941516.192043:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941516.192051:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241164160, portal 25 00000100:00000001:3.0:1773941516.192054:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941516.192055:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941516.192062:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241164160/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941532 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.192149:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941516.192152:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941516.192154:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a6e8. 00000100:00000200:3.0:1773941516.192157:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241164160, offset 0 00000400:00000200:3.0:1773941516.192162:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941516.192169:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941516.192176:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941516.192180:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941516.192183:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941516.192186:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941516.192188:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941516.192191:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[156, 155] 00000400:00000200:3.0:1773941516.192193:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941516.192203:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [262:262:256:4294967295] 192.168.204.153@tcp LPNI seq info [157:157:8:4294967295] 00000400:00000200:3.0:1773941516.192207:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941516.192216:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941516.192221:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941516.192225:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69d00. 00000800:00000200:3.0:1773941516.192231:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941516.192237:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941516.192240:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69d00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941516.192338:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941516.192338:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941516.192345:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69d00. 00000400:00000200:3.0:1773941516.192346:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941516.192351:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294956555 00000400:00000200:2.0:1773941516.192352:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941516.192357:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294955277 00000400:00000200:2.0:1773941516.192362:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:3.0:1773941516.192365:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294956555 00000400:00000200:2.0:1773941516.192368:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a6e8 00000100:00000001:3.0:1773941516.192370:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294955277 00000400:00000010:2.0:1773941516.192371:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a6e8. 00000100:00000001:3.0:1773941516.192375:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941516.192378:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941516.192378:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941516.192384:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241164160/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941532 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.192387:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941516.192390:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941516.192396:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:2.0:1773941516.192399:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941516.192400:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941516.192405:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941516.192407:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241164160/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941532 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941516.192414:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941516.192416:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941516.193355:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941516.193362:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941516.193364:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941516.193365:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941516.193369:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941516.193373:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941516.193377:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941516.193380:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941516.193383:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941516.193386:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321607f80 00000400:00000200:2.0:1773941516.193389:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941516.193399:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x125d [1] + 224 00000800:00000001:2.0:1773941516.193404:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941516.193410:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941516.193411:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941516.193415:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941516.193419:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941516.193423:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241164160/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941532 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941516.193430:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241164160/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941532 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941516.193442:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941516.193447:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941516.193449:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941516.193491:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941516.193498:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941516.193500:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941516.193502:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941516.193506:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941516.193508:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941516.193511:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941516.193518:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241164160/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941532 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941516.193528:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241164160/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941532 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.193536:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941516.193539:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941516.193541:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941516.193544:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941516.193546:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941516.193549:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941516.193551:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941516.193557:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=95 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941516.193560:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941516.193563:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941516.193567:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941516.193570:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941516.193573:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941516.193576:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941516.193580:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941516.193584:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941516.193587:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941516.193594:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941516.193596:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941516.193597:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941516.193600:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241164160/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941532 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.193606:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941516.193607:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941516.193608:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941516.193610:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941516.193612:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941516.193613:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941516.193615:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941516.193616:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941516.193616:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941516.193617:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941516.193619:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941516.193621:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941516.193622:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941516.193624:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941516.193626:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 285 00000400:00000200:3.0:1773941516.193629:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941516.193632:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941516.193635:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941516.193637:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941516.193639:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941516.193640:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941516.193642:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941516.193644:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941516.193645:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941516.193647:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941516.193649:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941516.193650:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941516.193652:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941516.193656:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941516.193657:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941516.193659:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941516.193661:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941516.193664:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941516.193665:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941516.193667:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941516.193669:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941516.193671:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941516.193672:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941516.193673:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941516.193676:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277100. 00000020:00000040:3.0:1773941516.193678:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941516.193680:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941516.193681:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941516.193683:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941516.193684:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941516.193686:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941516.193689:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90263000. 00000100:00000001:3.0:1773941516.193691:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941516.193695:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90277100 x1860112241164288/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941516.193701:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941516.193724:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941516.193728:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cc92d01] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90277100 x1860112241164288/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.193742:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941516.193744:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941516.193746:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941516.193748:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941516.193751:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941516.193752:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941516.193758:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941516.193758:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941516.193763:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941516.193765:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941516.193766:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941516.193769:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941516.193772:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241164160/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941532 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.193780:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:1.0:1773941516.193781:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90277100 x1860112241164288/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.193783:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941516.193786:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941516.193789:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941516.193796:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241164160/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941532 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941516.193800:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941516.193804:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941516.193809:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241164160:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000001:1.0:1773941516.193811:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1773941516.193813:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1773941516.193815:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941516.193816:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941516.193821:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941516.193823:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941516.193825:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241164160/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941532 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941516.193828:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941516.193832:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941516.193835:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00100000:1.0:1773941516.193837:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241164288:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941516.193838:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941516.193841:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:3.0:1773941516.193845:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:1.0:1773941516.193847:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:3.0:1773941516.193848:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941516.193851:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:1.0:1773941516.193852:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1773941516.193853:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 00000020:00000001:3.0:1773941516.193856:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 02000000:00000001:1.0:1773941516.193856:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941516.193863:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 02000000:00000001:1.0:1773941516.193863:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00010000:3.0:1773941516.193868:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1dfd lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:1.0:1773941516.193870:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0737c00. 02000000:00000001:1.0:1773941516.193875:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941516.193886:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941516.193887:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000010:1.0:1773941516.193891:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00010000:00010000:3.0:1773941516.193892:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1dfd lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941516.193896:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:3.0:1773941516.193898:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1dfd lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941516.193907:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941516.193908:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:1.0:1773941516.193908:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00010000:00010000:3.0:1773941516.193912:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1dfd lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:1.0:1773941516.193914:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241164288, portal 25 00010000:00000001:3.0:1773941516.193918:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:1.0:1773941516.193920:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:3.0:1773941516.193921:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941516.193923:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109629696 : -128249599921920 : ffff8b5b90277100) 00010000:00000001:3.0:1773941516.193925:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:3.0:1773941516.193930:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1dfd lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:1.0:1773941516.193935:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90277100 x1860112241164288/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941526 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1773941516.193936:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:3.0:1773941516.193940:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941516.193943:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1dfd lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941516.193957:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:3.0:1773941516.193959:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:1.0:1773941516.193959:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1773941516.193962:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941516.193965:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000100:00000040:1.0:1773941516.193966:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000020:00000001:3.0:1773941516.193969:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000010:1.0:1773941516.193971:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f14c8. 00010000:00000001:3.0:1773941516.193972:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000100:00000200:1.0:1773941516.193976:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241164288, offset 0 00010000:00000001:3.0:1773941516.193983:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:1.0:1773941516.193985:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:3.0:1773941516.193986:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1dfd lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941516.193994:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:3.0:1773941516.193996:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941516.194000:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1dfd lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941516.194002:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 10000000:01000000:3.0:1773941516.194009:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:1.0:1773941516.194011:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00000040:3.0:1773941516.194013:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941516.194015:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941516.194018:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00010000:3.0:1773941516.194019:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941516.194022:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:1.0:1773941516.194025:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:3.0:1773941516.194026:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941516.194028:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941516.194030:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:1.0:1773941516.194032:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:3.0:1773941516.194033:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941516.194039:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1dfd from hash 00010000:00000001:3.0:1773941516.194043:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:1.0:1773941516.194044:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[156, 157] 00010000:00000001:3.0:1773941516.194045:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941516.194047:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941516.194049:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:3.0:1773941516.194052:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:1.0:1773941516.194054:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:3.0:1773941516.194055:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:3.0:1773941516.194060:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941516.194062:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941516.194064:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941516.194065:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [263:263:256:4294967295] 192.168.204.143@tcp LPNI seq info [158:158:8:4294967295] 10000000:00000001:3.0:1773941516.194066:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941516.194071:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1dfd lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941516.194078:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941516.194079:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941516.194082:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941516.194084:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941516.194086:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941516.194091:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941516.194092:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:1.0:1773941516.194092:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:3.0:1773941516.194095:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941516.194097:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1dfd lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:1.0:1773941516.194104:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941516.194109:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b000. 00010000:00000040:3.0:1773941516.194115:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962900 count: 0 00000800:00000200:1.0:1773941516.194118:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:3.0:1773941516.194128:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:1.0:1773941516.194129:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:3.0:1773941516.194131:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941516.194136:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000800:00000200:1.0:1773941516.194137:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b000 type 1, nob 616 niov 1 nkiov 1 00000100:00000040:3.0:1773941516.194141:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241164160/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941532 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.194147:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941516.194149:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941516.194152:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 00000100:00000001:1.0:1773941516.194153:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941516.194155:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941516.194157:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941516.194160:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941516.194163:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000200:1.0:1773941516.194163:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 02000000:00000010:3.0:1773941516.194167:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2600. 00000100:00000001:1.0:1773941516.194168:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294956557 02000000:00000001:3.0:1773941516.194171:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941516.194174:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941516.194176:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:1.0:1773941516.194177:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294956555 00000100:00000001:3.0:1773941516.194179:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941516.194180:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941516.194181:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941516.194184:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941516.194188:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941516.194192:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068600. 00000100:00000001:1.0:1773941516.194192:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294956557 00000020:00000010:3.0:1773941516.194198:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b839e2600. 00000100:00000001:1.0:1773941516.194200:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294956555 00000020:00000010:3.0:1773941516.194204:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b839e3a00. 00000100:00000001:1.0:1773941516.194205:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000040:00000010:3.0:1773941516.194210:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b839e3000. 00000100:00000001:1.0:1773941516.194211:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941516.194212:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000020:00000010:3.0:1773941516.194215:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000800:00000010:2.0:1773941516.194216:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b000. 00000100:00000001:1.0:1773941516.194219:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941516.194221:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b839e2a00. 00000400:00000200:2.0:1773941516.194222:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1773941516.194224:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 80000000:00000010:3.0:1773941516.194227:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00000400:00000200:2.0:1773941516.194230:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941516.194234:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f14c8 00000100:00000001:1.0:1773941516.194234:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000400:00000010:2.0:1773941516.194237:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f14c8. 00040000:00000010:3.0:1773941516.194239:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b84d8a200. 00000100:00000001:1.0:1773941516.194239:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941516.194241:0:2347:0:(events.c:34:request_out_callback()) Process entered 20000000:00000010:3.0:1773941516.194247:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b84d8bc00. 00000100:00000001:1.0:1773941516.194247:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:2.0:1773941516.194248:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277100 x1860112241164288/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941526 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941516.194251:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000040:3.0:1773941516.194255:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000100:00000001:1.0:1773941516.194256:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941516.194261:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941516.194262:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941516.194263:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000010:3.0:1773941516.194267:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b839e3a00. 00000040:00000010:3.0:1773941516.194272:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b839e3000. 00000100:00000040:2.0:1773941516.194274:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90277100 x1860112241164288/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941526 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1773941516.194275:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941516.194281:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b839e2a00. 80000000:00000010:3.0:1773941516.194285:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00000100:00000001:2.0:1773941516.194292:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941516.194295:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00040000:00000010:3.0:1773941516.194342:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b84d8a200. 20000000:00000010:3.0:1773941516.194345:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b84d8bc00. 00000020:00000010:3.0:1773941516.194348:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b839e2600. 10000000:00000010:3.0:1773941516.194350:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068600. 10000000:01000000:3.0:1773941516.194352:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941516.194354:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941516.194356:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941516.194358:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941516.194359:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941516.194360:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941516.194361:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941516.195628:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941516.195636:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941516.195637:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941516.195639:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941516.195643:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941516.195652:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941516.195657:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941516.195659:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941516.195662:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941516.195665:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608000 00000400:00000200:2.0:1773941516.195668:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941516.195671:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x126d [1] + 0 00000800:00000001:2.0:1773941516.195675:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941516.195680:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941516.195682:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941516.195690:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941516.195693:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941516.195697:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277100 x1860112241164288/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941526 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941516.195724:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90277100 x1860112241164288/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941526 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941516.195737:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941516.195742:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941516.195745:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941516.195766:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941516.195768:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941516.195772:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941516.195773:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941516.195775:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941516.195778:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941516.195780:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941516.195782:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941516.195786:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277100 x1860112241164288/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941526 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941516.195793:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277100 x1860112241164288/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941526 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941516.195797:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941516.195798:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941516.195800:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941516.195801:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941516.195803:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941516.195804:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941516.195805:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941516.195808:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=95 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941516.195810:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941516.195812:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941516.195813:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941516.195820:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941516.195822:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941516.195824:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941516.195827:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90277100 x1860112241164288/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941526 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941516.195832:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941516.195833:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941516.195835:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941516.195836:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1773941516.195840:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:1.0:1773941516.195841:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941516.195842:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941516.195843:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941516.195845:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941516.195846:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941516.195848:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:1.0:1773941516.195851:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941516 v=0 (0 0 0 0) 00000100:00001000:1.0:1773941516.195854:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:1.0:1773941516.195859:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:1.0:1773941516.195862:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941516.195863:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:1.0:1773941516.195865:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941516.195867:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941516.195870:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:1.0:1773941516.195873:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:1.0:1773941516.195875:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:1.0:1773941516.195878:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:1.0:1773941516.195882:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:1.0:1773941516.195885:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941516.196028:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941516.196031:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941516.196038:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90277100 x1860112241164288/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941526 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941516.196042:0:12239:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:0.0:1773941516.196048:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241164288:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00080000:1.0:1773941516.196052:0:12239:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:0.0:1773941516.196054:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941516.196056:0:12239:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941516.196058:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941516.196061:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90277100 x1860112241164288/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941526 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941516.196065:0:12239:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941516.196068:0:12239:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941516.196072:0:12239:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:0.0:1773941516.196073:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:1.0:1773941516.196076:0:12239:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:0.0:1773941516.196077:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941516.196080:0:12239:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941516.196081:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0737c00. 00000001:00000001:1.0:1773941516.196086:0:12239:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:0.0:1773941516.196087:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941516.196088:0:12239:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:0.0:1773941516.196090:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:1.0:1773941516.196091:0:12239:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941516.196093:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 10000000:00000001:1.0:1773941516.196094:0:12239:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941516.196098:0:12239:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:0.0:1773941516.196098:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941516.196103:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90263000. 00000100:00080000:1.0:1773941516.196105:0:12239:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941516.196109:0:12239:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:0.0:1773941516.196110:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941516.196111:0:12239:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:0.0:1773941516.196115:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941516.196116:0:12239:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941516.196120:0:12239:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000010:0.0:1773941516.196121:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277100. 00000100:00000001:1.0:1773941516.196122:0:12239:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941516.196125:0:12239:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941516.196127:0:12239:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:0.0:1773941516.196128:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941516.196132:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941516.196134:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941516.196140:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941516.196141:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941516.196143:0:12239:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:0.0:1773941516.196145:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000001:1.0:1773941516.196146:0:12239:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941516.196149:0:12239:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941516.196149:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:02000000:1.0:1773941516.196155:0:12239:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941516.196157:0:12239:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941516.196159:0:12239:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941516.196161:0:12239:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941516.196162:0:12239:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941516.196164:0:12239:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941516.201731:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962900. 00010000:00000010:3.1:1773941516.201742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941517.151956:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941517.151963:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941517.151968:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941517.663805:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941517.663811:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941517.663815:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941517.663818:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941517.663819:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941517.663820:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941517.663823:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941517.663825:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941517.663827:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941517.663829:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941517.663830:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941517.663839:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941517.663856:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941517.663860:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941517.663862:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941517.663866:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941517.663873:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b839e2a00. 00000100:00000001:3.0:1773941517.663878:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941517.663882:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941517.663884:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941517.663891:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962c00. 00010000:00000001:3.0:1773941517.663895:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941517.663898:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941517.663901:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941517.663903:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1e04 to hash 00000020:00000001:3.0:1773941517.663905:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941517.663906:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941517.663909:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941517.663911:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941517.663914:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941517.663917:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e04 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941517.663925:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e04 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941517.663934:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e04 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941517.663939:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941517.663940:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941517.663945:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941517.663947:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941517.663949:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941517.663950:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941517.663954:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941517.663960:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941517.663962:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941517.663963:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941517.663972:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241164416/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941517.663979:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941517.663981:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941517.663984:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941517.663986:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941517.663987:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941517.663990:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241164416:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941517.663992:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941517.663997:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941517.663998:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941517.664000:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941517.664004:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941517.664005:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941517.664010:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941517.664013:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941517.664020:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941517.664027:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241164416, portal 25 00000100:00000001:3.0:1773941517.664029:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941517.664030:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941517.664036:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241164416/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941533 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941517.664045:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941517.664048:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941517.664051:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a550. 00000100:00000200:3.0:1773941517.664054:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241164416, offset 0 00000400:00000200:3.0:1773941517.664060:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941517.664069:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941517.664077:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941517.664082:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941517.664092:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941517.664096:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941517.664098:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941517.664100:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[158, 157] 00000400:00000200:3.0:1773941517.664103:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941517.664106:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [264:264:256:4294967295] 192.168.204.153@tcp LPNI seq info [159:159:8:4294967295] 00000400:00000200:3.0:1773941517.664109:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941517.664118:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941517.664123:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941517.664128:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69d00. 00000800:00000200:3.0:1773941517.664133:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941517.664140:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941517.664143:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69d00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941517.664159:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941517.664162:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941517.664165:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294958027 00000100:00000001:3.0:1773941517.664171:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294956557 00000100:00000001:3.0:1773941517.664175:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294958027 00000100:00000001:3.0:1773941517.664177:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294956557 00000100:00000001:3.0:1773941517.664180:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941517.664182:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941517.664184:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941517.664185:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941517.664188:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941517.664190:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941517.664192:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941517.664342:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941517.664351:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69d00. 00000400:00000200:2.0:1773941517.664360:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941517.664369:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941517.664379:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a550 00000400:00000010:2.0:1773941517.664384:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a550. 00000100:00000001:2.0:1773941517.664392:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941517.664401:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241164416/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941533 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941517.664422:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941517.664427:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241164416/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941533 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941517.664473:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941517.664476:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941517.665288:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941517.665305:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941517.665308:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941517.665312:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941517.665318:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941517.665323:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941517.665329:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941517.665333:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941517.665338:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941517.665341:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608080 00000400:00000200:2.0:1773941517.665345:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941517.665350:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x1275 [1] + 224 00000800:00000001:2.0:1773941517.665357:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941517.665365:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941517.665367:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941517.665370:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941517.665374:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941517.665377:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241164416/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941533 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941517.665385:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241164416/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941533 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941517.665401:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941517.665409:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941517.665411:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941517.665468:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941517.665477:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941517.665479:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941517.665482:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941517.665487:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941517.665497:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941517.665502:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941517.665509:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241164416/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941533 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941517.665521:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241164416/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941533 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941517.665529:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941517.665531:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941517.665533:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941517.665536:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941517.665538:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941517.665541:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941517.665543:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941517.665548:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=96 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941517.665551:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941517.665554:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941517.665559:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941517.665562:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941517.665565:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941517.665569:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941517.665573:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941517.665577:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941517.665581:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941517.665583:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941517.665585:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941517.665587:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941517.665591:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241164416/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941533 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941517.665599:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941517.665600:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941517.665602:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941517.665614:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941517.665617:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941517.665618:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941517.665620:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941517.665622:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941517.665623:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941517.665624:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941517.665626:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941517.665628:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941517.665630:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941517.665633:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941517.665637:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 286 00000400:00000200:3.0:1773941517.665643:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941517.665647:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941517.665652:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941517.665655:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941517.665657:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941517.665660:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941517.665663:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941517.665665:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941517.665667:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941517.665669:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941517.665671:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941517.665674:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941517.665677:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941517.665679:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941517.665681:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941517.665683:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941517.665687:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941517.665691:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941517.665693:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941517.665696:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941517.665699:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941517.665731:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941517.665734:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941517.665735:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941517.665739:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277480. 00000020:00000040:3.0:1773941517.665741:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941517.665744:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941517.665745:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941517.665746:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941517.665748:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941517.665750:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941517.665754:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90263000. 00000100:00000001:3.0:1773941517.665757:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941517.665762:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90277480 x1860112241164544/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941517.665768:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941517.665770:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941517.665773:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000944fb8d1] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90277480 x1860112241164544/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941517.665789:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941517.665791:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941517.665792:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941517.665794:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941517.665796:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941517.665797:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941517.665799:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941517.665800:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941517.665802:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941517.665805:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241164416/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941533 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941517.665810:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941517.665812:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941517.665814:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941517.665816:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941517.665824:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241164416/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941533 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941517.665830:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241164416:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941517.665831:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941517.665835:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941517.665837:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941517.665838:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941517.665842:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941517.665842:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1773941517.665846:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941517.665853:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241164416/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941533 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941517.665855:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90277480 x1860112241164544/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941517.665863:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941517.665867:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941517.665872:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941517.665874:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941517.665880:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:3.0:1773941517.665881:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:3.0:1773941517.665883:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941517.665885:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941517.665886:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 00000020:00000001:3.0:1773941517.665887:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941517.665890:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000100:00000001:0.0:1773941517.665893:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941517.665895:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e04 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941517.665900:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941517.665905:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1773941517.665906:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:0.0:1773941517.665911:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941517.665913:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e04 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1773941517.665918:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241164544:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:3.0:1773941517.665922:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:3.0:1773941517.665926:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e04 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941517.665927:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941517.665932:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941517.665935:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941517.665939:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941517.665946:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0c00. 00010000:00000001:3.0:1773941517.665948:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000001:0.0:1773941517.665950:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941517.665953:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:3.0:1773941517.665958:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e04 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941517.665958:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941517.665965:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00000001:3.0:1773941517.665972:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000010:0.0:1773941517.665974:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00010000:00000001:3.0:1773941517.665976:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1773941517.665979:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241164544, portal 25 00010000:00000001:3.0:1773941517.665981:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000001:0.0:1773941517.665984:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00010000:3.0:1773941517.665986:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e04 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941517.665988:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109630592 : -128249599921024 : ffff8b5b90277480) 00000100:00000040:0.0:1773941517.665998:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90277480 x1860112241164544/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941527 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1773941517.666001:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:3.0:1773941517.666007:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:0.0:1773941517.666009:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 10000000:00010000:3.0:1773941517.666012:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e04 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941517.666013:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941517.666017:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6cf68. 00000100:00000200:0.0:1773941517.666022:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241164544, offset 0 00010000:00000001:3.0:1773941517.666028:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:0.0:1773941517.666029:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941517.666033:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:0.0:1773941517.666037:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:3.0:1773941517.666038:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941517.666042:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000400:00000200:0.0:1773941517.666042:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941517.666047:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1773941517.666048:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000200:0.0:1773941517.666051:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:3.0:1773941517.666055:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000200:0.0:1773941517.666056:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941517.666061:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:3.0:1773941517.666063:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:0.0:1773941517.666066:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[158, 159] 00010000:00010000:3.0:1773941517.666068:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e04 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941517.666072:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941517.666079:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [265:265:256:4294967295] 192.168.204.143@tcp LPNI seq info [160:160:8:4294967295] 10000000:00000001:3.0:1773941517.666084:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941517.666088:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e04 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941517.666092:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:01000000:3.0:1773941517.666100:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:3.0:1773941517.666111:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:0.0:1773941517.666112:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 10000000:00000001:3.0:1773941517.666115:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941517.666121:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00010000:3.0:1773941517.666123:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000800:00000010:0.0:1773941517.666126:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b91e1e300. 00010000:00000001:3.0:1773941517.666128:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:3.0:1773941517.666133:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000800:00000200:0.0:1773941517.666134:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:3.0:1773941517.666138:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941517.666142:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941517.666145:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941517.666148:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000020:00000040:3.0:1773941517.666150:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1e04 from hash 00000800:00000200:0.0:1773941517.666152:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b91e1e300 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:3.0:1773941517.666157:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941517.666160:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941517.666162:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941517.666163:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:0.0:1773941517.666165:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941517.666166:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941517.666170:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:0.0:1773941517.666171:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941517.666175:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294958029 00010000:00000001:3.0:1773941517.666176:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941517.666181:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941517.666185:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294958027 00010000:00000001:3.0:1773941517.666186:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941517.666193:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941517.666194:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294958029 00010000:00010000:3.0:1773941517.666201:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e04 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941517.666202:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294958027 00000100:00000001:0.0:1773941517.666207:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941517.666212:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941517.666216:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941517.666218:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941517.666220:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1773941517.666222:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941517.666225:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941517.666228:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:0.0:1773941517.666228:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:3.0:1773941517.666234:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941517.666234:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:3.0:1773941517.666238:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941517.666240:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:2.0:1773941517.666241:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:3.0:1773941517.666242:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941517.666243:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941517.666248:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941517.666250:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e04 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:2.0:1773941517.666251:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b91e1e300. 00000100:00000001:0.0:1773941517.666251:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941517.666266:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000040:3.0:1773941517.666267:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962c00 count: 0 00000400:00000200:2.0:1773941517.666279:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00010000:00000001:3.0:1773941517.666285:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941517.666286:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6cf68 00010000:00000001:3.0:1773941517.666290:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941517.666294:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000010:2.0:1773941517.666299:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6cf68. 00000100:00000001:2.0:1773941517.666308:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941517.666317:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277480 x1860112241164544/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941527 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941517.666326:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941517.666330:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90277480 x1860112241164544/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941527 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941517.666338:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941517.666340:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1773941517.666463:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241164416/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941533 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941517.666470:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941517.666472:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941517.666474:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941517.666477:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941517.666478:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941517.666479:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941517.666481:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941517.666483:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b839e2a00. 02000000:00000001:3.0:1773941517.666485:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941517.666486:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941517.666487:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:3.0:1773941517.666489:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941517.666491:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941517.666492:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941517.666494:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941517.666497:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941517.666500:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068600. 00000020:00000010:3.0:1773941517.666503:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b839e2a00. 00000020:00000010:3.0:1773941517.666506:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b839e3000. 00000040:00000010:3.0:1773941517.666509:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b839e3a00. 00000020:00000010:3.0:1773941517.666513:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941517.666516:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b839e2600. 80000000:00000010:3.0:1773941517.666520:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941517.666525:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b84e8e400. 20000000:00000010:3.0:1773941517.666529:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5ba0ab8a00. 10000000:00000040:3.0:1773941517.666533:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:3.0:1773941517.666534:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941517.666538:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b839e3000. 00000040:00000010:3.0:1773941517.666541:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b839e3a00. 00000020:00000010:3.0:1773941517.666543:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941517.666545:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b839e2600. 80000000:00000010:3.0:1773941517.666547:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941517.666555:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b84e8e400. 20000000:00000010:3.0:1773941517.666557:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5ba0ab8a00. 00000020:00000010:3.0:1773941517.666559:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b839e2a00. 10000000:00000010:3.0:1773941517.666561:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068600. 10000000:01000000:3.0:1773941517.666563:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941517.666565:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941517.666567:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941517.666568:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941517.666569:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941517.666570:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941517.666571:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941517.667741:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941517.667753:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941517.667756:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941517.667759:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941517.667764:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941517.667770:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941517.667777:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941517.667781:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941517.667786:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941517.667790:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608100 00000400:00000200:2.0:1773941517.667795:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941517.667800:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1285 [1] + 0 00000800:00000001:2.0:1773941517.667806:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941517.667815:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941517.667818:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941517.667821:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941517.667826:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941517.667832:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277480 x1860112241164544/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941527 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941517.667844:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90277480 x1860112241164544/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941527 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941517.667864:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941517.667881:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941517.667885:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941517.667893:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941517.667896:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941517.667900:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941517.667901:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941517.667904:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941517.667908:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941517.667910:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941517.667913:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941517.667919:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277480 x1860112241164544/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941527 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941517.667944:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277480 x1860112241164544/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941527 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941517.667952:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941517.667954:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941517.667957:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941517.667959:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941517.667960:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941517.667962:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941517.667965:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941517.667969:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=96 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941517.667972:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941517.667975:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941517.667977:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941517.667979:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941517.667981:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941517.667984:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941517.667988:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90277480 x1860112241164544/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941527 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941517.667994:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941517.667995:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941517.667996:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941517.667998:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941517.668003:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941517.668004:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941517.668005:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941517.668006:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941517.668008:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941517.668009:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941517.668011:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941517.668014:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941517 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941517.668016:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941517.668021:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941517.668027:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941517.668029:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941517.668030:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941517.668031:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941517.668034:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941517.668036:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941517.668038:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941517.668040:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941517.668043:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941517.668044:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941517.668330:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941517.668336:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941517.668344:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90277480 x1860112241164544/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941527 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941517.668356:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241164544:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941517.668359:0:12240:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:1.0:1773941517.668363:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941517.668367:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1773941517.668373:0:12240:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941517.668379:0:12240:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:1.0:1773941517.668382:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90277480 x1860112241164544/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941527 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941517.668383:0:12240:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941517.668386:0:12240:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941517.668390:0:12240:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941517.668394:0:12240:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:1.0:1773941517.668396:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941517.668401:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941517.668401:0:12240:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941517.668406:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0c00. 00000001:00000001:0.0:1773941517.668406:0:12240:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941517.668410:0:12240:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:1.0:1773941517.668412:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941517.668413:0:12240:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941517.668416:0:2351:0:(genops.c:1236:class_import_put()) Process entered 10000000:00000001:0.0:1773941517.668417:0:12240:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941517.668420:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941517.668422:0:12240:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:1.0:1773941517.668426:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941517.668429:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90263000. 00000100:00080000:0.0:1773941517.668429:0:12240:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941517.668445:0:12240:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941517.668449:0:12240:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:1.0:1773941517.668450:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1773941517.668452:0:12240:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941517.668453:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941517.668458:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277480. 00000100:00080000:0.0:1773941517.668458:0:12240:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941517.668465:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941517.668465:0:12240:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941517.668467:0:12240:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:1.0:1773941517.668469:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000040:0.0:1773941517.668472:0:12240:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:1.0:1773941517.668474:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941517.668480:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941517.668481:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941517.668484:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941517.668485:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941517.668490:0:12240:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941517.668493:0:12240:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941517.668496:0:12240:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941517.668502:0:12240:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941517.668513:0:12240:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941517.668516:0:12240:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941517.668519:0:12240:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941517.668521:0:12240:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941517.668523:0:12240:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941517.671741:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962c00. 00010000:00000010:3.1:1773941517.671749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:0.0:1773941518.175830:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941518.175835:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941518.175838:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941519.071819:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941519.071827:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941519.071832:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941519.071835:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941519.071836:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941519.071838:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941519.071842:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941519.071844:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941519.071846:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941519.071848:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941519.071850:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941519.071860:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941519.071864:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941519.071868:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941519.071871:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941519.071876:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941519.071882:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5ba0ab8800. 00000100:00000001:3.0:1773941519.071888:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941519.071891:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941519.071893:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941519.071900:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962900. 00010000:00000001:3.0:1773941519.071905:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941519.071907:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941519.071911:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941519.071913:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1e0b to hash 00000020:00000001:3.0:1773941519.071930:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941519.071931:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941519.071934:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941519.071937:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941519.071940:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941519.071943:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e0b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941519.071951:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e0b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941519.071960:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e0b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941519.071965:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941519.071966:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941519.071972:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941519.071973:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941519.071975:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941519.071977:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941519.071980:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941519.071984:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941519.071985:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941519.071987:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941519.071996:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241164672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941519.072003:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941519.072004:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941519.072007:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941519.072009:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941519.072011:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941519.072013:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241164672:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941519.072020:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941519.072025:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941519.072027:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941519.072029:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941519.072032:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941519.072034:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941519.072039:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941519.072042:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941519.072048:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941519.072053:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241164672, portal 25 00000100:00000001:3.0:1773941519.072054:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941519.072055:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941519.072059:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241164672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941535 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941519.072064:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941519.072066:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941519.072068:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a660. 00000100:00000200:3.0:1773941519.072070:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241164672, offset 0 00000400:00000200:3.0:1773941519.072075:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941519.072082:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941519.072088:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941519.072093:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941519.072096:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941519.072099:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941519.072101:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941519.072104:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[160, 159] 00000400:00000200:3.0:1773941519.072106:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941519.072109:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [266:266:256:4294967295] 192.168.204.153@tcp LPNI seq info [161:161:8:4294967295] 00000400:00000200:3.0:1773941519.072112:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941519.072119:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941519.072123:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941519.072130:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69500. 00000800:00000200:3.0:1773941519.072135:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941519.072141:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941519.072145:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69500 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941519.072163:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941519.072168:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941519.072171:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294959435 00000100:00000001:3.0:1773941519.072177:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294958029 00000100:00000001:3.0:1773941519.072183:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294959435 00000100:00000001:3.0:1773941519.072186:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294958029 00000100:00000001:3.0:1773941519.072189:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941519.072192:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941519.072195:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941519.072197:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941519.072200:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941519.072202:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941519.072205:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941519.072307:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941519.072313:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69500. 00000400:00000200:2.0:1773941519.072317:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941519.072322:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941519.072326:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a660 00000400:00000010:2.0:1773941519.072328:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a660. 00000100:00000001:2.0:1773941519.072332:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941519.072336:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241164672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941535 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941519.072344:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941519.072346:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241164672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941535 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941519.072352:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941519.072353:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941519.073477:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941519.073486:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941519.073489:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941519.073490:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941519.073495:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941519.073499:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941519.073504:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941519.073507:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941519.073510:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941519.073513:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608180 00000400:00000200:2.0:1773941519.073516:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941519.073519:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x128d [1] + 224 00000800:00000001:2.0:1773941519.073524:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941519.073530:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941519.073532:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941519.073535:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941519.073538:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941519.073547:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241164672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941535 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941519.073555:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241164672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941535 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941519.073568:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941519.073575:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941519.073578:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941519.073589:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941519.073596:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941519.073597:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941519.073600:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941519.073603:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941519.073605:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941519.073607:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941519.073612:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241164672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941535 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941519.073620:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241164672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941535 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941519.073625:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941519.073626:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941519.073628:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941519.073631:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941519.073633:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941519.073641:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941519.073643:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941519.073647:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=98 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941519.073650:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941519.073652:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941519.073655:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941519.073657:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941519.073660:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941519.073662:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941519.073665:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941519.073668:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941519.073671:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941519.073673:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941519.073674:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941519.073676:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941519.073678:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241164672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941535 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941519.073683:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941519.073684:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941519.073685:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941519.073687:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941519.073689:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941519.073690:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941519.073692:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941519.073692:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941519.073693:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941519.073694:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941519.073695:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941519.073696:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941519.073698:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941519.073700:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941519.073719:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 288 00000400:00000200:3.0:1773941519.073723:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941519.073731:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941519.073734:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941519.073736:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941519.073738:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941519.073740:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941519.073742:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941519.073743:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941519.073745:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941519.073747:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941519.073749:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941519.073750:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941519.073753:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941519.073754:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941519.073755:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941519.073756:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941519.073758:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941519.073760:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941519.073762:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941519.073763:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941519.073765:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941519.073767:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941519.073769:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941519.073770:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941519.073773:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90276a00. 00000020:00000040:3.0:1773941519.073774:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941519.073777:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941519.073778:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941519.073779:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941519.073781:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941519.073782:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941519.073785:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90262000. 00000100:00000001:3.0:1773941519.073788:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941519.073792:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90276a00 x1860112241164800/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941519.073801:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941519.073803:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941519.073807:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e6b7771] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90276a00 x1860112241164800/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941519.073829:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941519.073831:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941519.073833:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941519.073835:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941519.073839:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941519.073840:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941519.073842:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941519.073843:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941519.073845:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941519.073848:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241164672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941535 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941519.073854:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941519.073856:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941519.073858:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941519.073859:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941519.073863:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241164672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941535 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941519.073868:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241164672:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941519.073871:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941519.073872:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941519.073877:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941519.073879:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941519.073879:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941519.073884:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941519.073884:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941519.073887:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241164672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941535 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941519.073896:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941519.073896:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90276a00 x1860112241164800/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941519.073904:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941519.073909:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941519.073911:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941519.073917:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:3.0:1773941519.073918:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:1.0:1773941519.073920:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941519.073922:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:1.0:1773941519.073924:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1773941519.073925:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:1.0:1773941519.073928:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1773941519.073929:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 02000000:00000001:1.0:1773941519.073933:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941519.073934:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941519.073939:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000100:00100000:1.0:1773941519.073940:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241164800:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941519.073946:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00010000:3.0:1773941519.073947:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e0b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:1.0:1773941519.073949:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941519.073953:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941519.073959:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:1.0:1773941519.073959:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00010000:3.0:1773941519.073964:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e0b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941519.073971:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:3.0:1773941519.073973:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e0b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:1.0:1773941519.073974:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0736c00. 02000000:00000001:1.0:1773941519.073980:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941519.073982:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941519.073985:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:3.0:1773941519.073987:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e0b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941519.073989:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941519.073996:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941519.073998:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1773941519.073998:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00010000:00000001:3.0:1773941519.074005:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:3.0:1773941519.074007:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e0b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:1.0:1773941519.074007:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941519.074015:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241164800, portal 25 00010000:00010000:3.0:1773941519.074016:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 00000100:00000001:1.0:1773941519.074020:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941519.074022:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109627904 : -128249599923712 : ffff8b5b90276a00) 10000000:00000001:3.0:1773941519.074026:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941519.074029:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e0b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:1.0:1773941519.074033:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90276a00 x1860112241164800/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941529 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941519.074037:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:3.0:1773941519.074040:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941519.074042:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941519.074044:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000020:00000001:3.0:1773941519.074048:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941519.074053:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000100:00000001:1.0:1773941519.074056:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:3.0:1773941519.074057:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:3.0:1773941519.074061:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e0b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:1.0:1773941519.074063:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 10000000:00000001:3.0:1773941519.074069:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000010:1.0:1773941519.074070:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1ee0. 10000000:00010000:3.0:1773941519.074071:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e0b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:1.0:1773941519.074077:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241164800, offset 0 10000000:01000000:3.0:1773941519.074078:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:3.0:1773941519.074084:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941519.074085:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941519.074087:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941519.074090:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:1.0:1773941519.074090:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941519.074093:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941519.074095:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941519.074097:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941519.074099:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941519.074100:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000040:3.0:1773941519.074101:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1e0b from hash 00010000:00000001:3.0:1773941519.074105:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941519.074106:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941519.074108:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941519.074110:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000400:00000200:1.0:1773941519.074110:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:3.0:1773941519.074113:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941519.074117:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:1.0:1773941519.074119:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:3.0:1773941519.074120:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:1.0:1773941519.074125:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:3.0:1773941519.074126:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941519.074129:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941519.074131:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941519.074133:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:3.0:1773941519.074135:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e0b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941519.074140:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:3.0:1773941519.074144:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941519.074148:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:1.0:1773941519.074150:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[160, 161] 00010000:00000001:3.0:1773941519.074151:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941519.074154:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941519.074156:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941519.074158:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941519.074161:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:1.0:1773941519.074162:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00010000:3.0:1773941519.074163:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e0b lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941519.074172:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [267:267:256:4294967295] 192.168.204.143@tcp LPNI seq info [162:162:8:4294967295] 00010000:00000040:3.0:1773941519.074173:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962900 count: 0 00000400:00000200:1.0:1773941519.074183:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941519.074185:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941519.074188:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941519.074191:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941519.074199:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241164672/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941535 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941519.074201:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000100:00000001:3.0:1773941519.074216:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000800:00000200:1.0:1773941519.074219:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 02000000:00000001:3.0:1773941519.074220:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941519.074224:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 00000800:00000010:1.0:1773941519.074226:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b000. 02000000:00000001:3.0:1773941519.074227:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941519.074231:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941519.074232:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000800:00000200:1.0:1773941519.074236:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000020:00000001:3.0:1773941519.074237:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941519.074241:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5ba0ab8800. 02000000:00000001:3.0:1773941519.074245:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941519.074248:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000800:00000200:1.0:1773941519.074249:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000100:00000010:3.0:1773941519.074251:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000800:00000200:1.0:1773941519.074253:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b000 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:3.0:1773941519.074254:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941519.074256:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941519.074257:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941519.074266:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941519.074268:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941519.074271:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068600. 00000020:00000010:3.0:1773941519.074274:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab8800. 00000100:00000001:1.0:1773941519.074277:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941519.074279:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab9a00. 00000040:00000010:3.0:1773941519.074284:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab9600. 00000020:00000010:3.0:1773941519.074287:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000400:00000200:1.0:1773941519.074292:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000010:3.0:1773941519.074293:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab9c00. 00000100:00000001:1.0:1773941519.074296:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294959437 80000000:00000010:3.0:1773941519.074298:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00000100:00000001:1.0:1773941519.074305:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294959435 00040000:00000010:3.0:1773941519.074306:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab9200. 20000000:00000010:3.0:1773941519.074312:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5ba0ab8400. 00000100:00000001:1.0:1773941519.074314:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294959437 10000000:00000040:3.0:1773941519.074318:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000100:00000001:1.0:1773941519.074320:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294959435 10000000:00000001:3.0:1773941519.074322:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941519.074328:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1773941519.074329:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab9a00. 00000040:00000010:3.0:1773941519.074334:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab9600. 00000100:00000001:1.0:1773941519.074335:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941519.074338:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000100:00000001:1.0:1773941519.074341:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941519.074344:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab9c00. 00000800:00000200:2.0:1773941519.074345:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1773941519.074346:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 80000000:00000010:3.0:1773941519.074348:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00000800:00000010:2.0:1773941519.074350:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b000. 00040000:00000010:3.0:1773941519.074354:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab9200. 00000100:00000001:1.0:1773941519.074354:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:2.0:1773941519.074356:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1773941519.074357:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 20000000:00000010:3.0:1773941519.074360:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5ba0ab8400. 00000100:00000001:1.0:1773941519.074363:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:3.0:1773941519.074365:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab8800. 00000400:00000200:2.0:1773941519.074367:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:1.0:1773941519.074368:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000010:3.0:1773941519.074369:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068600. 00000100:00000001:1.0:1773941519.074372:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:3.0:1773941519.074376:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000400:00000200:2.0:1773941519.074376:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1ee0 00000100:00000001:1.0:1773941519.074376:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1773941519.074381:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1ee0. 10000000:00000001:3.0:1773941519.074382:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941519.074386:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000100:00000001:2.0:1773941519.074386:0:2347:0:(events.c:34:request_out_callback()) Process entered 10000000:00000001:3.0:1773941519.074391:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941519.074393:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941519.074396:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 00000100:00000200:2.0:1773941519.074397:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90276a00 x1860112241164800/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941529 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:3.0:1773941519.074398:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:2.0:1773941519.074426:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941519.074431:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90276a00 x1860112241164800/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941529 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941519.074439:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941519.074441:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941519.075787:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941519.075793:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941519.075795:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941519.075796:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941519.075799:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941519.075803:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941519.075806:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941519.075808:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941519.075811:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941519.075813:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608200 00000400:00000200:2.0:1773941519.075815:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941519.075823:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x129d [1] + 0 00000800:00000001:2.0:1773941519.075827:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941519.075831:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941519.075833:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941519.075835:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941519.075838:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941519.075841:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90276a00 x1860112241164800/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941529 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941519.075848:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90276a00 x1860112241164800/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941529 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941519.075864:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941519.075875:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941519.075878:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941519.075900:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941519.075902:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941519.075905:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941519.075906:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941519.075908:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941519.075911:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941519.075913:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941519.075915:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941519.075919:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90276a00 x1860112241164800/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941529 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941519.075927:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90276a00 x1860112241164800/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941529 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941519.075932:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941519.075933:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941519.075934:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941519.075935:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941519.075936:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941519.075937:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941519.075944:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941519.075947:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=98 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941519.075949:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941519.075951:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941519.075953:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941519.075955:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941519.075956:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941519.075958:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941519.075961:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90276a00 x1860112241164800/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941529 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941519.075965:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941519.075967:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941519.075968:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941519.075969:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1773941519.075973:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:1.0:1773941519.075974:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941519.075975:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941519.075977:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941519.075978:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941519.075980:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941519.075981:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:1.0:1773941519.075984:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941519 v=0 (0 0 0 0) 00000100:00001000:1.0:1773941519.075986:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:1.0:1773941519.075990:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:1.0:1773941519.075992:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941519.075993:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:1.0:1773941519.075994:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941519.075995:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941519.075998:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:1.0:1773941519.075999:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:1.0:1773941519.076001:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:1.0:1773941519.076003:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:1.0:1773941519.076005:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:1.0:1773941519.076011:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941519.076245:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941519.076249:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941519.076256:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90276a00 x1860112241164800/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941529 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941519.076266:0:12241:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:0.0:1773941519.076269:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241164800:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00080000:1.0:1773941519.076275:0:12241:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:0.0:1773941519.076278:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941519.076279:0:12241:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941519.076282:0:12241:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:0.0:1773941519.076283:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:01000000:1.0:1773941519.076284:0:12241:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941519.076288:0:12241:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941519.076291:0:12241:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000040:0.0:1773941519.076293:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90276a00 x1860112241164800/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941529 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:1.0:1773941519.076295:0:12241:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941519.076299:0:12241:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941519.076300:0:12241:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941519.076302:0:12241:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941519.076306:0:12241:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941519.076309:0:12241:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941519.076313:0:12241:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941519.076315:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1773941519.076316:0:12241:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941519.076320:0:12241:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:0.0:1773941519.076320:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1773941519.076324:0:12241:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941519.076327:0:12241:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 02000000:00000010:0.0:1773941519.076327:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0736c00. 00000100:00000001:1.0:1773941519.076332:0:12241:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941519.076334:0:12241:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941519.076335:0:12241:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 02000000:00000001:0.0:1773941519.076335:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941519.076340:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941519.076344:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941519.076347:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941519.076349:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90262000. 00000100:00080000:1.0:1773941519.076350:0:12241:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941519.076353:0:12241:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941519.076353:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941519.076354:0:12241:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:0.0:1773941519.076358:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:02000000:1.0:1773941519.076360:0:12241:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000010:0.0:1773941519.076361:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90276a00. 00000100:00000001:1.0:1773941519.076364:0:12241:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941519.076366:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000020:00000001:1.0:1773941519.076367:0:12241:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941519.076369:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1773941519.076370:0:12241:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941519.076373:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1773941519.076374:0:12241:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941519.076377:0:12241:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941519.076380:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941519.076383:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941519.076392:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941519.076394:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941519.079744:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962900. 00010000:00000010:3.1:1773941519.079751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941519.199952:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941519.199958:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941519.199962:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941519.263807:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (294) 00000020:00000001:3.0:1773941520.223831:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941520.223836:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941520.223857:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941520.863836:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941520.863844:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941520.863849:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941520.863852:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941520.863853:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941520.863860:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941520.863864:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941520.863866:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941520.863869:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941520.863872:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941520.863874:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941520.863886:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941520.863891:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941520.863896:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941520.863899:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941520.863904:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941520.863911:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5ba0ab9a00. 00000100:00000001:3.0:1773941520.863918:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941520.863923:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941520.863926:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941520.863933:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962c00. 00010000:00000001:3.0:1773941520.863939:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941520.863942:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941520.863947:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941520.863950:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1e12 to hash 00000020:00000001:3.0:1773941520.863952:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941520.863954:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941520.863958:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941520.863962:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941520.863965:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941520.863969:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e12 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941520.863982:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e12 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941520.864009:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e12 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941520.864016:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941520.864018:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941520.864025:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941520.864028:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941520.864031:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941520.864032:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941520.864038:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941520.864042:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941520.864045:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941520.864047:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941520.864058:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241164928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941520.864068:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941520.864071:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941520.864075:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941520.864078:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941520.864082:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941520.864086:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241164928:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941520.864090:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941520.864096:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941520.864099:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941520.864107:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941520.864112:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941520.864115:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941520.864123:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941520.864127:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941520.864133:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941520.864140:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241164928, portal 25 00000100:00000001:3.0:1773941520.864142:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941520.864149:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941520.864156:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241164928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941536 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941520.864164:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941520.864168:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941520.864171:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6af68. 00000100:00000200:3.0:1773941520.864174:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241164928, offset 0 00000400:00000200:3.0:1773941520.864182:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941520.864190:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941520.864198:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941520.864203:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941520.864208:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941520.864213:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941520.864216:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941520.864220:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[162, 161] 00000400:00000200:3.0:1773941520.864225:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941520.864229:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [268:268:256:4294967295] 192.168.204.153@tcp LPNI seq info [163:163:8:4294967295] 00000400:00000200:3.0:1773941520.864234:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941520.864243:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941520.864250:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941520.864255:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69f00. 00000800:00000200:3.0:1773941520.864263:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941520.864271:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941520.864275:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69f00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941520.864293:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941520.864298:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941520.864301:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294961227 00000100:00000001:3.0:1773941520.864307:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294959437 00000100:00000001:3.0:1773941520.864313:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294961227 00000100:00000001:3.0:1773941520.864318:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294959437 00000100:00000001:3.0:1773941520.864328:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941520.864332:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941520.864334:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941520.864336:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941520.864339:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941520.864342:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941520.864345:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941520.864430:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941520.864435:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69f00. 00000400:00000200:2.0:1773941520.864439:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941520.864444:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941520.864448:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6af68 00000400:00000010:2.0:1773941520.864455:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6af68. 00000100:00000001:2.0:1773941520.864460:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941520.864466:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241164928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941536 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941520.864474:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941520.864477:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241164928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941536 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941520.864483:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941520.864484:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941520.865434:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941520.865444:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941520.865447:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941520.865448:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941520.865453:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941520.865463:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941520.865468:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941520.865471:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941520.865474:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941520.865477:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608280 00000400:00000200:2.0:1773941520.865481:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941520.865485:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x12a5 [1] + 224 00000800:00000001:2.0:1773941520.865490:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941520.865497:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941520.865499:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941520.865501:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941520.865505:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941520.865509:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241164928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941536 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941520.865517:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241164928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941536 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941520.865530:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941520.865537:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941520.865540:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941520.865548:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941520.865553:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941520.865555:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941520.865557:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941520.865560:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941520.865562:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941520.865564:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941520.865568:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241164928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941536 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941520.865576:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241164928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941536 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941520.865581:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941520.865582:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941520.865584:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941520.865586:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941520.865587:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941520.865589:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941520.865590:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941520.865594:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=99 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941520.865597:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941520.865599:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941520.865602:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941520.865604:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941520.865607:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941520.865609:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941520.865612:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941520.865615:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941520.865625:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941520.865627:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941520.865628:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941520.865630:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941520.865632:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241164928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941536 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941520.865654:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941520.865655:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941520.865657:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941520.865659:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941520.865661:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941520.865662:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941520.865663:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941520.865664:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941520.865665:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941520.865666:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941520.865667:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941520.865669:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941520.865670:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941520.865672:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941520.865673:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 289 00000400:00000200:3.0:1773941520.865677:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941520.865680:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941520.865683:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941520.865685:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941520.865687:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941520.865689:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941520.865692:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941520.865693:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941520.865695:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941520.865696:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941520.865698:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941520.865715:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941520.865720:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941520.865722:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941520.865723:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941520.865724:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941520.865726:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941520.865729:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941520.865730:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941520.865748:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941520.865750:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941520.865752:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941520.865754:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941520.865755:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941520.865758:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277800. 00000020:00000040:3.0:1773941520.865760:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941520.865762:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941520.865764:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941520.865765:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941520.865767:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941520.865769:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941520.865772:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90262000. 00000100:00000001:3.0:1773941520.865774:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941520.865778:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90277800 x1860112241165056/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941520.865784:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941520.865786:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941520.865790:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fd608f47] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90277800 x1860112241165056/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941520.865803:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941520.865805:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941520.865806:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941520.865808:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941520.865811:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941520.865817:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941520.865819:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941520.865821:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941520.865823:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941520.865829:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241164928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941536 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941520.865838:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941520.865841:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941520.865843:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941520.865845:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941520.865848:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1773941520.865852:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241164928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941536 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941520.865853:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1773941520.865863:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241164928:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941520.865866:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1773941520.865873:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941520.865876:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941520.865880:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000040:0.0:1773941520.865880:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90277800 x1860112241165056/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941520.865884:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941520.865892:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241164928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941536 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941520.865897:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941520.865901:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941520.865906:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1773941520.865908:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941520.865910:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1773941520.865913:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 02000000:00000001:0.0:1773941520.865915:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941520.865919:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00100000:0.0:1773941520.865923:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241165056:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941520.865924:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:3.0:1773941520.865931:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:0.0:1773941520.865932:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:3.0:1773941520.865936:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:0.0:1773941520.865937:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1773941520.865941:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:0.0:1773941520.865941:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941520.865945:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 02000000:00000001:0.0:1773941520.865946:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1773941520.865951:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 02000000:00000010:0.0:1773941520.865953:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c3000. 00010000:00000001:3.0:1773941520.865955:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 02000000:00000001:0.0:1773941520.865958:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941520.865963:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e12 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941520.865964:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941520.865969:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941520.865977:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941520.865982:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241165056, portal 25 00010000:00000001:3.0:1773941520.865984:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941520.865986:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941520.865991:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109631488 : -128249599920128 : ffff8b5b90277800) 00010000:00010000:3.0:1773941520.865993:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e12 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941520.865999:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90277800 x1860112241165056/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941520.866010:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:3.0:1773941520.866015:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e12 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941520.866016:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941520.866022:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941520.866027:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c990. 00010000:00000001:3.0:1773941520.866030:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000200:0.0:1773941520.866033:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241165056, offset 0 00010000:00000001:3.0:1773941520.866034:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:3.0:1773941520.866040:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e12 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941520.866041:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941520.866049:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941520.866053:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941520.866057:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941520.866061:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:3.0:1773941520.866064:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:0.0:1773941520.866065:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941520.866069:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00010000:3.0:1773941520.866070:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e12 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941520.866073:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941520.866079:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941520.866084:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[162, 163] 00010000:00010000:3.0:1773941520.866091:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:0.0:1773941520.866091:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941520.866097:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [269:269:256:4294967295] 192.168.204.143@tcp LPNI seq info [164:164:8:4294967295] 10000000:00000001:3.0:1773941520.866098:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941520.866102:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00010000:3.0:1773941520.866103:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e12 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941520.866112:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:3.0:1773941520.866121:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000800:00000200:0.0:1773941520.866123:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941520.866125:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941520.866129:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941520.866131:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000800:00000010:0.0:1773941520.866131:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b91e1e300. 00000020:00000001:3.0:1773941520.866138:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941520.866142:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000800:00000200:0.0:1773941520.866143:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:0.0:1773941520.866154:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:3.0:1773941520.866155:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000200:0.0:1773941520.866158:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b91e1e300 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:3.0:1773941520.866161:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e12 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:3.0:1773941520.866170:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941520.866174:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e12 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941520.866177:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:3.0:1773941520.866182:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:0.0:1773941520.866187:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000040:3.0:1773941520.866189:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000001:0.0:1773941520.866191:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294961229 10000000:00000001:3.0:1773941520.866193:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941520.866197:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294961227 00010000:00010000:3.0:1773941520.866200:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:0.0:1773941520.866202:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294961229 00010000:00000001:3.0:1773941520.866205:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:0.0:1773941520.866207:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294961227 00000100:00000001:0.0:1773941520.866211:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1773941520.866212:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:0.0:1773941520.866214:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941520.866216:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:0.0:1773941520.866217:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941520.866220:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:0.0:1773941520.866221:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1773941520.866226:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941520.866229:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:3.0:1773941520.866232:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1e12 from hash 00000100:00000001:0.0:1773941520.866232:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1773941520.866236:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:3.0:1773941520.866238:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:0.0:1773941520.866239:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:3.0:1773941520.866243:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941520.866243:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1773941520.866244:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941520.866247:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941520.866248:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1773941520.866249:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b91e1e300. 00010000:00000001:3.0:1773941520.866250:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:3.0:1773941520.866254:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:2.0:1773941520.866255:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:3.0:1773941520.866260:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:2.0:1773941520.866263:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00010000:00000001:3.0:1773941520.866265:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941520.866267:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c990 00010000:00000001:3.0:1773941520.866269:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:2.0:1773941520.866270:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c990. 00010000:00000001:3.0:1773941520.866273:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941520.866274:0:2347:0:(events.c:34:request_out_callback()) Process entered 10000000:00000001:3.0:1773941520.866278:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941520.866281:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277800 x1860112241165056/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1773941520.866284:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e12 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941520.866296:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000001:3.0:1773941520.866300:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000040:2.0:1773941520.866301:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90277800 x1860112241165056/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941530 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941520.866303:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941520.866306:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941520.866309:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:2.0:1773941520.866324:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941520.866325:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1773941520.866364:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941520.866366:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941520.866367:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941520.866370:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e12 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:3.0:1773941520.866396:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962c00 count: 0 00010000:00000001:3.0:1773941520.866407:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941520.866409:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941520.866412:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941520.866417:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241164928/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941536 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941520.866425:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941520.866428:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941520.866431:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941520.866434:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941520.866436:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941520.866438:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941520.866440:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941520.866443:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5ba0ab9a00. 02000000:00000001:3.0:1773941520.866446:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941520.866447:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941520.866449:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:3.0:1773941520.866453:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941520.866454:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941520.866456:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941520.866459:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941520.866462:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941520.866466:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068600. 00000020:00000010:3.0:1773941520.866470:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab9a00. 00000020:00000010:3.0:1773941520.866474:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab9800. 00000040:00000010:3.0:1773941520.866479:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab9e00. 00000020:00000010:3.0:1773941520.866484:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941520.866488:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab8200. 80000000:00000010:3.0:1773941520.866493:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941520.866506:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab9000. 20000000:00000010:3.0:1773941520.866511:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5ba0ab8000. 10000000:00000040:3.0:1773941520.866516:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:3.0:1773941520.866519:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941520.866524:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab9800. 00000040:00000010:3.0:1773941520.866528:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab9e00. 00000020:00000010:3.0:1773941520.866531:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941520.866535:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab8200. 80000000:00000010:3.0:1773941520.866538:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941520.866542:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab9000. 20000000:00000010:3.0:1773941520.866545:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5ba0ab8000. 00000020:00000010:3.0:1773941520.866548:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab9a00. 10000000:00000010:3.0:1773941520.866550:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068600. 10000000:01000000:3.0:1773941520.866553:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941520.866556:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941520.866559:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941520.866561:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941520.866563:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941520.866565:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941520.866567:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941520.867674:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941520.867681:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941520.867683:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941520.867685:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941520.867689:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941520.867692:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941520.867695:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941520.867697:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941520.867700:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941520.867726:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608300 00000400:00000200:2.0:1773941520.867732:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941520.867737:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x12b5 [1] + 0 00000800:00000001:2.0:1773941520.867742:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941520.867748:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941520.867750:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941520.867753:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941520.867757:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941520.867761:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277800 x1860112241165056/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941530 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941520.867768:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90277800 x1860112241165056/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941530 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941520.867779:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941520.867783:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941520.867785:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941520.867803:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941520.867805:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941520.867809:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941520.867810:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941520.867812:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941520.867815:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941520.867816:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941520.867818:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941520.867822:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277800 x1860112241165056/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941530 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941520.867829:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277800 x1860112241165056/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941530 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941520.867834:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941520.867835:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941520.867837:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941520.867839:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941520.867840:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941520.867841:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941520.867843:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941520.867845:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=99 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941520.867847:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941520.867849:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941520.867850:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941520.867852:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941520.867854:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941520.867855:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941520.867858:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90277800 x1860112241165056/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941530 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941520.867863:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941520.867864:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941520.867865:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941520.867867:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941520.867870:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941520.867872:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941520.867873:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941520.867879:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941520.867881:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941520.867882:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941520.867884:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941520.867887:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941520 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941520.867889:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941520.867893:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941520.867895:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941520.867896:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941520.867898:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941520.867899:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941520.867902:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941520.867904:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941520.867905:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941520.867908:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941520.867910:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941520.867912:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941520.868117:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941520.868120:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941520.868126:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90277800 x1860112241165056/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941530 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941520.868136:0:12242:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:1.0:1773941520.868145:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241165056:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00080000:0.0:1773941520.868149:0:12242:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:1.0:1773941520.868151:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941520.868154:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:0.0:1773941520.868154:0:12242:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941520.868158:0:12242:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000040:1.0:1773941520.868160:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90277800 x1860112241165056/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941530 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:01000000:0.0:1773941520.868161:0:12242:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941520.868165:0:12242:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941520.868169:0:12242:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:1.0:1773941520.868170:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941520.868173:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941520.868175:0:12242:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941520.868176:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c3000. 00000001:00000001:0.0:1773941520.868180:0:12242:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:1.0:1773941520.868181:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941520.868184:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941520.868184:0:12242:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:1.0:1773941520.868188:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941520.868190:0:12242:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941520.868191:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941520.868193:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90262000. 10000000:00000001:0.0:1773941520.868195:0:12242:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941520.868196:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941520.868202:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941520.868202:0:12242:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000010:1.0:1773941520.868205:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277800. 00000100:00000001:1.0:1773941520.868208:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941520.868209:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941520.868209:0:12242:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941520.868214:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941520.868216:0:12242:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941520.868218:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941520.868221:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941520.868221:0:12242:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941520.868223:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941520.868225:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941520.868231:0:12242:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941520.868235:0:12242:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941520.868238:0:12242:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941520.868240:0:12242:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941520.868241:0:12242:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941520.868254:0:12242:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941520.868255:0:12242:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941520.868256:0:12242:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941520.868259:0:12242:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941520.868261:0:12242:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941520.868263:0:12242:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941520.868265:0:12242:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941520.868267:0:12242:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941520.868268:0:12242:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941520.871748:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962c00. 00010000:00000010:3.1:1773941520.871755:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941521.247863:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941521.247867:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941521.247871:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941522.271870:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941522.271878:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941522.271885:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941522.719801:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941522.719809:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941522.719814:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941522.719817:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941522.719819:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941522.719820:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941522.719823:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941522.719825:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941522.719828:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941522.719853:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941522.719854:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941522.719863:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941522.719868:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941522.719873:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941522.719877:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941522.719882:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941522.719890:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5ba0ab9400. 00000100:00000001:3.0:1773941522.719896:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941522.719901:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941522.719904:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941522.719912:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962c00. 00010000:00000001:3.0:1773941522.719918:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941522.719921:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941522.719926:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941522.719929:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1e19 to hash 00000020:00000001:3.0:1773941522.719932:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941522.719933:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941522.719936:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941522.719940:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941522.719944:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941522.719948:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e19 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941522.719959:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e19 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941522.719970:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e19 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941522.719978:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941522.719980:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941522.719987:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941522.719990:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941522.719999:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941522.720000:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941522.720006:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941522.720011:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941522.720013:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941522.720015:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941522.720026:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241165184/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941522.720035:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941522.720038:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941522.720042:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941522.720044:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941522.720047:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941522.720051:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241165184:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941522.720055:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941522.720061:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941522.720063:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941522.720066:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941522.720071:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941522.720074:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941522.720081:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941522.720086:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941522.720093:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941522.720100:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241165184, portal 25 00000100:00000001:3.0:1773941522.720102:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941522.720104:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941522.720109:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241165184/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941538 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941522.720117:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941522.720120:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941522.720123:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a7f8. 00000100:00000200:3.0:1773941522.720126:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241165184, offset 0 00000400:00000200:3.0:1773941522.720133:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941522.720147:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941522.720156:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941522.720161:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941522.720166:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941522.720170:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941522.720173:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941522.720177:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[164, 163] 00000400:00000200:3.0:1773941522.720182:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941522.720186:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [270:270:256:4294967295] 192.168.204.153@tcp LPNI seq info [165:165:8:4294967295] 00000400:00000200:3.0:1773941522.720191:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941522.720201:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941522.720208:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941522.720213:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69700. 00000800:00000200:3.0:1773941522.720220:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941522.720227:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941522.720232:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69700 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941522.720252:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941522.720256:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941522.720259:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294963083 00000100:00000001:3.0:1773941522.720266:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294961229 00000100:00000001:3.0:1773941522.720272:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294963083 00000100:00000001:3.0:1773941522.720275:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294961229 00000100:00000001:3.0:1773941522.720279:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941522.720283:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941522.720286:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941522.720288:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941522.720291:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941522.720295:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941522.720298:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941522.720406:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941522.720410:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69700. 00000400:00000200:2.0:1773941522.720415:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941522.720420:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941522.720424:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a7f8 00000400:00000010:2.0:1773941522.720433:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a7f8. 00000100:00000001:2.0:1773941522.720437:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941522.720441:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241165184/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941538 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941522.720449:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941522.720452:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241165184/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941538 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941522.720457:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941522.720458:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941522.721380:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941522.721390:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941522.721393:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941522.721394:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941522.721398:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941522.721403:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941522.721407:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941522.721410:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941522.721413:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941522.721416:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608380 00000400:00000200:2.0:1773941522.721419:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941522.721424:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x12bd [1] + 224 00000800:00000001:2.0:1773941522.721429:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941522.721436:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941522.721438:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941522.721441:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941522.721444:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941522.721448:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241165184/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941538 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941522.721456:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241165184/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941538 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941522.721470:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941522.721482:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941522.721484:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941522.721505:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941522.721512:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941522.721568:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941522.721571:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941522.721576:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941522.721578:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941522.721581:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941522.721587:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241165184/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941538 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941522.721599:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241165184/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941538 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941522.721606:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941522.721608:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941522.721611:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941522.721615:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941522.721617:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941522.721619:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941522.721622:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941522.721628:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=101 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941522.721631:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941522.721634:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941522.721636:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941522.721639:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941522.721641:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941522.721644:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941522.721647:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941522.721650:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941522.721652:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941522.721654:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941522.721656:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941522.721657:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941522.721660:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241165184/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941538 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941522.721665:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941522.721671:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941522.721673:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941522.721675:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941522.721677:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941522.721678:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941522.721679:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941522.721680:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941522.721681:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941522.721682:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941522.721683:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941522.721685:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941522.721686:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941522.721688:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941522.721690:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 291 00000400:00000200:3.0:1773941522.721694:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941522.721696:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941522.721699:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941522.721719:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941522.721721:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941522.721723:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941522.721725:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941522.721727:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941522.721728:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941522.721730:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941522.721732:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941522.721734:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941522.721735:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941522.721737:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941522.721738:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941522.721739:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941522.721741:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941522.721743:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941522.721745:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941522.721750:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941522.721753:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941522.721754:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941522.721756:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941522.721758:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941522.721761:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90276680. 00000020:00000040:3.0:1773941522.721763:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941522.721765:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941522.721766:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941522.721768:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941522.721770:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941522.721771:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941522.721775:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90263000. 00000100:00000001:3.0:1773941522.721777:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941522.721782:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90276680 x1860112241165312/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941522.721787:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941522.721789:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941522.721792:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ff4e98f] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90276680 x1860112241165312/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941522.721807:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941522.721810:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941522.721812:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941522.721815:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941522.721817:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941522.721819:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941522.721821:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941522.721824:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941522.721826:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941522.721830:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241165184/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941538 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941522.721839:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941522.721842:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941522.721849:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941522.721849:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941522.721851:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941522.721853:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941522.721855:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941522.721856:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241165184/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941538 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941522.721860:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90276680 x1860112241165312/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941522.721866:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241165184:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941522.721868:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941522.721870:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941522.721872:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1773941522.721873:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1773941522.721874:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941522.721875:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:1.0:1773941522.721876:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941522.721878:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941522.721879:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941522.721882:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241165184/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941538 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941522.721884:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241165312:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941522.721888:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1773941522.721890:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941522.721890:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941522.721892:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941522.721893:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 02000000:00000001:1.0:1773941522.721894:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1773941522.721896:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941522.721898:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000010:1.0:1773941522.721898:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735c00. 02000000:00000001:1.0:1773941522.721900:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941522.721901:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:3.0:1773941522.721904:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:1.0:1773941522.721904:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:3.0:1773941522.721906:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941522.721907:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 00000400:00000010:1.0:1773941522.721908:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000020:00000001:3.0:1773941522.721910:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941522.721912:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000010:1.0:1773941522.721912:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941522.721915:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241165312, portal 25 00010000:00010000:3.0:1773941522.721917:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e19 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941522.721918:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941522.721919:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109627008 : -128249599924608 : ffff8b5b90276680) 00000100:00000040:1.0:1773941522.721924:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90276680 x1860112241165312/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941532 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941522.721925:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:3.0:1773941522.721930:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e19 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941522.721931:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941522.721934:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941522.721936:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1880. 00010000:00000001:3.0:1773941522.721938:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000200:1.0:1773941522.721939:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241165312, offset 0 00010000:00010000:3.0:1773941522.721941:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e19 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941522.721944:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941522.721948:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941522.721952:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:3.0:1773941522.721953:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941522.721955:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:1.0:1773941522.721956:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:3.0:1773941522.721958:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e19 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941522.721959:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941522.721962:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941522.721964:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:3.0:1773941522.721965:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941522.721967:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941522.721968:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[164, 165] 00010000:00000001:3.0:1773941522.721971:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:1.0:1773941522.721971:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00010000:3.0:1773941522.721974:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e19 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941522.721975:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [271:271:256:4294967295] 192.168.204.143@tcp LPNI seq info [166:166:8:4294967295] 00000400:00000200:1.0:1773941522.721980:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:3.0:1773941522.721981:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:3.0:1773941522.721984:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941522.721987:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e19 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941522.721988:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:3.0:1773941522.721994:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:3.0:1773941522.721996:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941522.721997:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941522.721999:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000800:00000200:1.0:1773941522.721999:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000020:00000001:3.0:1773941522.722001:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941522.722004:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000800:00000010:1.0:1773941522.722004:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024be00. 00010000:00000001:3.0:1773941522.722007:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000200:1.0:1773941522.722009:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00010000:3.0:1773941522.722010:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e19 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:1.0:1773941522.722015:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 10000000:00000001:3.0:1773941522.722016:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941522.722019:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e19 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:1.0:1773941522.722019:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024be00 type 1, nob 616 niov 1 nkiov 1 10000000:01000000:3.0:1773941522.722025:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:3.0:1773941522.722032:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000001:1.0:1773941522.722032:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941522.722033:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941522.722036:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:1.0:1773941522.722036:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941522.722038:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:1.0:1773941522.722038:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294963085 00010000:00000001:3.0:1773941522.722041:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941522.722043:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:1.0:1773941522.722043:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294963083 00010000:00000001:3.0:1773941522.722046:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:1.0:1773941522.722047:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294963085 00010000:00000001:3.0:1773941522.722048:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941522.722052:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1e19 from hash 00000100:00000001:1.0:1773941522.722052:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294963083 00010000:00000001:3.0:1773941522.722055:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:1.0:1773941522.722056:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1773941522.722057:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941522.722059:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:1.0:1773941522.722059:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941522.722060:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:1.0:1773941522.722061:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941522.722062:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941522.722064:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:1.0:1773941522.722064:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1773941522.722067:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941522.722068:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941522.722070:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941522.722072:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941522.722072:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941522.722073:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00010000:3.0:1773941522.722076:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e19 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941522.722076:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941522.722077:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941522.722080:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941522.722082:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:1.0:1773941522.722082:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941522.722083:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941522.722085:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941522.722086:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941522.722088:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941522.722090:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941522.722092:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941522.722093:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00010000:3.0:1773941522.722095:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e19 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:2.0:1773941522.722097:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024be00. 00000400:00000200:2.0:1773941522.722104:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000040:3.0:1773941522.722107:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962c00 count: 0 00000400:00000200:2.0:1773941522.722110:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941522.722114:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1880 00000400:00000010:2.0:1773941522.722116:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1880. 00000100:00000001:2.0:1773941522.722119:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:3.0:1773941522.722122:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000200:2.0:1773941522.722123:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90276680 x1860112241165312/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941532 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941522.722125:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941522.722135:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941522.722137:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90276680 x1860112241165312/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941532 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941522.722142:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941522.722143:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1773941522.722167:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941522.722177:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241165184/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941538 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941522.722183:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941522.722185:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941522.722187:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941522.722189:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941522.722190:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941522.722191:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941522.722192:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941522.722194:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5ba0ab9400. 02000000:00000001:3.0:1773941522.722197:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941522.722198:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941522.722199:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:3.0:1773941522.722201:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941522.722202:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941522.722204:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941522.722206:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941522.722208:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941522.722211:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068600. 00000020:00000010:3.0:1773941522.722213:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab9400. 00000020:00000010:3.0:1773941522.722217:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab8600. 00000040:00000010:3.0:1773941522.722220:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab8e00. 00000020:00000010:3.0:1773941522.722223:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941522.722226:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab8a00. 80000000:00000010:3.0:1773941522.722230:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941522.722234:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab9800. 20000000:00000010:3.0:1773941522.722237:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5ba0ab9e00. 10000000:00000040:3.0:1773941522.722241:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:3.0:1773941522.722242:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941522.722246:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab8600. 00000040:00000010:3.0:1773941522.722248:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab8e00. 00000020:00000010:3.0:1773941522.722250:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941522.722253:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab8a00. 80000000:00000010:3.0:1773941522.722255:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941522.722262:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab9800. 20000000:00000010:3.0:1773941522.722264:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5ba0ab9e00. 00000020:00000010:3.0:1773941522.722265:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab9400. 10000000:00000010:3.0:1773941522.722267:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068600. 10000000:01000000:3.0:1773941522.722269:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941522.722271:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941522.722273:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941522.722274:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941522.722275:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941522.722276:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941522.722278:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941522.723487:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941522.723493:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941522.723494:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941522.723496:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941522.723499:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941522.723502:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941522.723506:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941522.723508:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941522.723510:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941522.723513:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608400 00000400:00000200:2.0:1773941522.723515:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941522.723534:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x12cd [1] + 0 00000800:00000001:2.0:1773941522.723538:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941522.723552:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941522.723554:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941522.723558:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941522.723561:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941522.723570:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90276680 x1860112241165312/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941532 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941522.723578:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90276680 x1860112241165312/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941532 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941522.723609:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941522.723613:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941522.723615:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941522.723643:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941522.723646:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941522.723649:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941522.723650:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941522.723651:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941522.723655:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941522.723656:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941522.723658:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941522.723662:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90276680 x1860112241165312/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941532 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941522.723670:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90276680 x1860112241165312/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941532 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941522.723685:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941522.723686:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941522.723688:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941522.723689:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941522.723690:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941522.723692:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941522.723693:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941522.723714:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=101 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941522.723716:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941522.723749:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941522.723751:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941522.723754:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941522.723757:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941522.723760:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941522.723768:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90276680 x1860112241165312/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941532 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941522.723793:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941522.723796:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941522.723798:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941522.723801:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1773941522.723806:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:1.0:1773941522.723809:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941522.723810:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941522.723813:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941522.723816:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941522.723819:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941522.723823:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:1.0:1773941522.723829:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941522 v=0 (0 0 0 0) 00000100:00001000:1.0:1773941522.723834:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:1.0:1773941522.723841:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:1.0:1773941522.723845:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941522.723847:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:1.0:1773941522.723849:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941522.723858:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941522.723863:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:1.0:1773941522.723865:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:1.0:1773941522.723868:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:1.0:1773941522.723872:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:1.0:1773941522.723876:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:1.0:1773941522.723879:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941522.724168:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941522.724173:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941522.724179:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90276680 x1860112241165312/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941532 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941522.724187:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241165312:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941522.724191:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941522.724193:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941522.724197:0:12243:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:0.0:1773941522.724197:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90276680 x1860112241165312/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941532 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941522.724206:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941522.724207:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:1.0:1773941522.724210:0:12243:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 02000000:00000010:0.0:1773941522.724211:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735c00. 00000100:00100000:1.0:1773941522.724218:0:12243:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:0.0:1773941522.724218:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941522.724220:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941522.724223:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941522.724224:0:12243:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000001:0.0:1773941522.724224:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 10000000:01000000:1.0:1773941522.724228:0:12243:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000010:0.0:1773941522.724229:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90263000. 02000000:00000001:0.0:1773941522.724234:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:1.0:1773941522.724235:0:12243:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:0.0:1773941522.724235:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:1.0:1773941522.724240:0:12243:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000010:0.0:1773941522.724243:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90276680. 00000001:00000001:1.0:1773941522.724246:0:12243:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941522.724247:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941522.724249:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941522.724252:0:12243:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:0.0:1773941522.724253:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941522.724255:0:12243:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:0.0:1773941522.724258:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000001:00000001:1.0:1773941522.724259:0:12243:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941522.724261:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941522.724263:0:12243:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941522.724264:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941522.724267:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941522.724269:0:12243:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941522.724274:0:12243:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941522.724277:0:12243:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941522.724278:0:12243:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941522.724280:0:12243:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941522.724282:0:12243:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941522.724284:0:12243:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941522.724285:0:12243:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941522.724286:0:12243:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941522.724297:0:12243:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941522.724299:0:12243:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941522.724300:0:12243:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941522.724302:0:12243:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941522.724304:0:12243:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941522.724306:0:12243:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941522.724313:0:12243:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941522.724315:0:12243:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941522.724316:0:12243:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941522.727739:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962c00. 00010000:00000010:3.1:1773941522.727747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941523.295893:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941523.295901:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941523.295907:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941523.935852:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941523.935867:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941523.935891:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941523.935895:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941523.935903:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941523.935905:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941523.935909:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941523.935912:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941523.935916:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941523.935921:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941523.935923:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941523.935942:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941523.935950:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941523.935956:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941523.935960:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941523.935967:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941523.935977:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5ba0ab9800. 00000100:00000001:3.0:1773941523.935985:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941523.935991:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941523.935995:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941523.936006:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962c00. 00010000:00000001:3.0:1773941523.936015:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941523.936041:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941523.936047:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941523.936051:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1e20 to hash 00000020:00000001:3.0:1773941523.936055:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941523.936057:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941523.936062:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941523.936067:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941523.936071:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941523.936077:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e20 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941523.936088:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e20 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941523.936102:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e20 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941523.936107:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941523.936109:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941523.936116:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941523.936118:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941523.936121:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941523.936122:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941523.936128:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941523.936133:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941523.936136:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941523.936138:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941523.936152:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241165440/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941523.936161:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941523.936163:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941523.936168:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941523.936170:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941523.936178:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941523.936182:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241165440:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941523.936186:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941523.936193:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941523.936196:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941523.936199:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941523.936203:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941523.936205:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941523.936214:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941523.936219:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941523.936226:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941523.936234:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241165440, portal 25 00000100:00000001:3.0:1773941523.936236:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941523.936237:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941523.936243:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241165440/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941539 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941523.936250:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941523.936253:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941523.936255:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6abb0. 00000100:00000200:3.0:1773941523.936258:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241165440, offset 0 00000400:00000200:3.0:1773941523.936265:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941523.936274:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941523.936283:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941523.936289:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941523.936293:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941523.936297:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941523.936299:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941523.936303:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[166, 165] 00000400:00000200:3.0:1773941523.936307:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941523.936311:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [272:272:256:4294967295] 192.168.204.153@tcp LPNI seq info [167:167:8:4294967295] 00000400:00000200:3.0:1773941523.936333:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941523.936344:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941523.936357:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941523.936362:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69c00. 00000800:00000200:3.0:1773941523.936370:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941523.936378:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941523.936383:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69c00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941523.936402:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941523.936407:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941523.936410:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294964299 00000100:00000001:3.0:1773941523.936417:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294963085 00000100:00000001:3.0:1773941523.936423:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294964299 00000100:00000001:3.0:1773941523.936427:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294963085 00000100:00000001:3.0:1773941523.936430:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941523.936433:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941523.936436:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941523.936438:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941523.936441:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941523.936445:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941523.936448:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941523.936558:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941523.936565:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69c00. 00000400:00000200:2.0:1773941523.936570:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941523.936578:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941523.936584:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6abb0 00000400:00000010:2.0:1773941523.936587:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6abb0. 00000100:00000001:2.0:1773941523.936593:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941523.936598:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241165440/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941539 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941523.936609:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941523.936613:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241165440/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941539 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941523.936620:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941523.936622:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941523.937919:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941523.937937:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941523.937940:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941523.937944:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941523.937951:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941523.937957:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941523.937965:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941523.937970:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941523.937976:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941523.937981:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608480 00000400:00000200:2.0:1773941523.937986:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941523.938013:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x12d5 [1] + 224 00000800:00000001:2.0:1773941523.938023:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941523.938034:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941523.938037:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941523.938041:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941523.938047:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941523.938053:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241165440/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941539 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941523.938065:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241165440/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941539 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941523.938085:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941523.938097:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941523.938102:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941523.938139:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941523.938155:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941523.938158:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941523.938163:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941523.938169:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941523.938171:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941523.938175:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941523.938185:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241165440/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941539 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941523.938194:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241165440/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941539 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941523.938200:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941523.938201:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941523.938219:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941523.938221:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941523.938223:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941523.938225:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941523.938227:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941523.938233:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=102 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941523.938236:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941523.938238:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941523.938241:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941523.938244:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941523.938247:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941523.938249:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941523.938253:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941523.938256:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941523.938258:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941523.938260:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941523.938261:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941523.938262:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941523.938265:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241165440/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941539 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941523.938270:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941523.938271:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941523.938272:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941523.938275:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941523.938277:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941523.938278:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941523.938280:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941523.938280:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941523.938281:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941523.938282:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941523.938284:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941523.938285:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941523.938287:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941523.938289:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941523.938294:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 293 00000400:00000200:3.0:1773941523.938301:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941523.938306:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941523.938308:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941523.938311:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941523.938326:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941523.938328:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941523.938331:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941523.938332:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941523.938334:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941523.938337:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941523.938338:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941523.938340:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941523.938342:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941523.938343:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941523.938345:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941523.938346:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941523.938348:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941523.938351:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941523.938352:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941523.938354:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941523.938357:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941523.938359:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941523.938360:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941523.938362:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941523.938366:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90274380. 00000020:00000040:3.0:1773941523.938369:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941523.938372:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941523.938373:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941523.938376:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941523.938378:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941523.938381:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941523.938390:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90263000. 00000100:00000001:3.0:1773941523.938395:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941523.938400:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90274380 x1860112241165568/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941523.938407:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941523.938410:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941523.938414:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2c15a31] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90274380 x1860112241165568/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941523.938428:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941523.938430:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941523.938431:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941523.938434:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941523.938436:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941523.938438:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941523.938439:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941523.938441:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941523.938443:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941523.938446:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241165440/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941539 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941523.938452:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941523.938454:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941523.938457:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941523.938458:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941523.938462:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941523.938462:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1773941523.938466:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241165440/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941539 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941523.938466:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941523.938474:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90274380 x1860112241165568/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941523.938478:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241165440:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941523.938487:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941523.938493:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941523.938493:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941523.938495:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941523.938497:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941523.938498:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1773941523.938501:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941523.938503:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941523.938506:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941523.938507:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241165440/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941539 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941523.938511:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90274380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241165568:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941523.938531:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1773941523.938535:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941523.938536:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:3.0:1773941523.938538:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 02000000:00000001:0.0:1773941523.938539:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941523.938543:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1773941523.938544:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941523.938546:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:3.0:1773941523.938549:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000010:0.0:1773941523.938549:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c3000. 02000000:00000001:0.0:1773941523.938553:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941523.938554:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941523.938557:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941523.938559:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 00000400:00000200:0.0:1773941523.938560:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:3.0:1773941523.938562:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000010:0.0:1773941523.938564:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00000001:3.0:1773941523.938565:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000010:0.0:1773941523.938571:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00010000:00010000:3.0:1773941523.938572:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e20 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:0.0:1773941523.938579:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241165568, portal 25 00010000:00000001:3.0:1773941523.938582:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941523.938583:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941523.938585:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109618048 : -128249599933568 : ffff8b5b90274380) 00010000:00010000:3.0:1773941523.938588:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e20 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941523.938592:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90274380 x1860112241165568/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941523.938601:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:3.0:1773941523.938605:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e20 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941523.938605:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:3.0:1773941523.938610:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941523.938612:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:3.0:1773941523.938615:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e20 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941523.938617:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941523.938621:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c770. 00010000:00000001:3.0:1773941523.938624:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000200:0.0:1773941523.938627:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241165568, offset 0 00010000:00000001:3.0:1773941523.938628:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941523.938633:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:0.0:1773941523.938637:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:3.0:1773941523.938638:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e20 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941523.938643:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941523.938651:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00010000:3.0:1773941523.938653:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:0.0:1773941523.938656:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00000001:3.0:1773941523.938657:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941523.938661:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e20 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941523.938662:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941523.938668:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:3.0:1773941523.938672:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:0.0:1773941523.938673:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:3.0:1773941523.938675:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941523.938678:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941523.938680:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000400:00000200:0.0:1773941523.938682:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[166, 167] 00000020:00000001:3.0:1773941523.938683:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941523.938687:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941523.938692:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:0.0:1773941523.938693:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00010000:3.0:1773941523.938694:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e20 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941523.938702:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [273:273:256:4294967295] 192.168.204.143@tcp LPNI seq info [168:168:8:4294967295] 10000000:00000001:3.0:1773941523.938729:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941523.938731:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e20 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1773941523.938735:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:3.0:1773941523.938737:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941523.938738:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941523.938740:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941523.938742:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:3.0:1773941523.938746:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941523.938748:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:0.0:1773941523.938749:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941523.938750:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941523.938753:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941523.938756:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1e20 from hash 00010000:00000001:3.0:1773941523.938759:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941523.938762:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941523.938765:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:0.0:1773941523.938765:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:3.0:1773941523.938772:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000800:00000200:0.0:1773941523.938772:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941523.938774:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941523.938777:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:3.0:1773941523.938779:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000010:0.0:1773941523.938779:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b91e1e300. 00010000:00000001:3.0:1773941523.938783:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941523.938795:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941523.938797:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941523.938800:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00010000:3.0:1773941523.938801:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e20 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941523.938808:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941523.938811:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:0.0:1773941523.938811:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:3.0:1773941523.938813:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941523.938817:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000800:00000200:0.0:1773941523.938818:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b91e1e300 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:3.0:1773941523.938819:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941523.938822:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941523.938823:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941523.938825:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e20 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:3.0:1773941523.938831:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962c00 count: 0 00000100:00000001:0.0:1773941523.938832:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941523.938837:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941523.938839:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294964302 00010000:00000001:3.0:1773941523.938842:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941523.938845:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941523.938846:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294964299 00000100:00000001:3.0:1773941523.938847:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941523.938852:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294964302 00000100:00000040:3.0:1773941523.938854:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241165440/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941539 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941523.938857:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294964299 00000100:00000001:0.0:1773941523.938862:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941523.938869:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941523.938869:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941523.938871:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941523.938873:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1773941523.938875:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1773941523.938876:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941523.938879:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941523.938880:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941523.938882:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941523.938885:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941523.938887:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5ba0ab9800. 00000100:00000001:0.0:1773941523.938888:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1773941523.938890:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1773941523.938891:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:3.0:1773941523.938893:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941523.938894:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941523.938896:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941523.938897:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941523.938899:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941523.938904:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:3.0:1773941523.938907:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941523.938908:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941523.938910:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941523.938912:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941523.938918:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941523.938921:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068600. 00000020:00000010:3.0:1773941523.938924:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab9800. 00000020:00000010:3.0:1773941523.938928:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab8a00. 00000040:00000010:3.0:1773941523.938932:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab8e00. 00000020:00000010:3.0:1773941523.938937:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941523.938940:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab8600. 80000000:00000010:3.0:1773941523.938945:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941523.938949:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab8200. 20000000:00000010:3.0:1773941523.938954:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5ba0ab9000. 10000000:00000040:3.0:1773941523.938959:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:3.0:1773941523.938961:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941523.938965:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab8a00. 00000040:00000010:3.0:1773941523.938967:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab8e00. 00000020:00000010:3.0:1773941523.938969:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941523.938974:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab8600. 80000000:00000010:3.0:1773941523.938976:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941523.938980:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab8200. 20000000:00000010:3.0:1773941523.938983:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5ba0ab9000. 00000020:00000010:3.0:1773941523.938985:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab9800. 10000000:00000010:3.0:1773941523.938988:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068600. 10000000:01000000:3.0:1773941523.938990:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941523.938992:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941523.938994:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941523.938996:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941523.938997:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941523.939000:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941523.939002:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000200:2.0:1773941523.939009:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941523.939015:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b91e1e300. 00000400:00000200:2.0:1773941523.939022:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941523.939029:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941523.939035:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c770 00000400:00000010:2.0:1773941523.939038:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c770. 00000100:00000001:2.0:1773941523.939042:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941523.939049:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90274380 x1860112241165568/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941523.939060:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941523.939065:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90274380 x1860112241165568/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941533 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941523.939074:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941523.939076:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941523.940457:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941523.940474:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941523.940480:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941523.940485:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941523.940495:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941523.940537:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941523.940548:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941523.940555:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941523.940561:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941523.940567:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608500 00000400:00000200:2.0:1773941523.940574:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941523.940582:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x12e5 [1] + 0 00000800:00000001:2.0:1773941523.940591:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941523.940603:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941523.940608:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941523.940615:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941523.940622:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941523.940631:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90274380 x1860112241165568/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941533 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941523.940650:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90274380 x1860112241165568/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941533 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941523.940676:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941523.940684:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941523.940688:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941523.940733:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941523.940738:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941523.940742:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941523.940744:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941523.940746:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941523.940749:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941523.940751:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941523.940753:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941523.940759:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90274380 x1860112241165568/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941533 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941523.940767:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90274380 x1860112241165568/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941533 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941523.940773:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941523.940774:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941523.940776:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941523.940777:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941523.940778:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941523.940780:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941523.940781:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941523.940784:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=102 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941523.940786:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941523.940788:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941523.940789:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941523.940791:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941523.940792:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941523.940794:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941523.940797:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90274380 x1860112241165568/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941533 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941523.940803:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941523.940804:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941523.940811:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941523.940813:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941523.940818:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941523.940819:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941523.940821:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941523.940822:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941523.940823:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941523.940825:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941523.940827:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941523.940829:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941523 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941523.940831:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941523.940835:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941523.940837:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941523.940838:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941523.940840:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941523.940841:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941523.940843:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941523.940845:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941523.940846:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941523.940849:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941523.940851:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941523.940853:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941523.941238:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941523.941242:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941523.941249:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90274380 x1860112241165568/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941533 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941523.941257:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90274380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241165568:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941523.941262:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941523.941264:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941523.941268:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90274380 x1860112241165568/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941533 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941523.941274:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941523.941276:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941523.941278:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c3000. 02000000:00000001:1.0:1773941523.941281:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941523.941282:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941523.941283:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941523.941285:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941523.941289:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90263000. 00000100:00000001:0.0:1773941523.941289:0:12245:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 02000000:00000001:1.0:1773941523.941293:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941523.941299:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941523.941300:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90274380. 00000100:00000001:1.0:1773941523.941305:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941523.941307:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941523.941309:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941523.941332:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941523.941334:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941523.941336:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941523.941340:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941523.941340:0:12245:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941523.941352:0:12245:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941523.941358:0:12245:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941523.941363:0:12245:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941523.941369:0:12245:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941523.941374:0:12245:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941523.941381:0:12245:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941523.941384:0:12245:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941523.941386:0:12245:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941523.941389:0:12245:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941523.941393:0:12245:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941523.941407:0:12245:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941523.941416:0:12245:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941523.941421:0:12245:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941523.941425:0:12245:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941523.941428:0:12245:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941523.941432:0:12245:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941523.941434:0:12245:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941523.941436:0:12245:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941523.941439:0:12245:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941523.941461:0:12245:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941523.941464:0:12245:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941523.941466:0:12245:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941523.941471:0:12245:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941523.941477:0:12245:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941523.941481:0:12245:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941523.941484:0:12245:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941523.941487:0:12245:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941523.941490:0:12245:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941523.944737:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962c00. 00010000:00000010:3.1:1773941523.944746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941524.319914:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941524.319964:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941524.319970:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941524.383802:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (299) 10000000:00000001:3.0:1773941524.959838:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941524.959846:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941524.959852:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941524.959855:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941524.959857:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941524.959859:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941524.959862:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941524.959864:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941524.959866:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941524.959870:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941524.959872:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941524.959883:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941524.959889:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941524.959894:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941524.959898:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941524.959904:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941524.959912:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5ba0ab8200. 00000100:00000001:3.0:1773941524.959919:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941524.959924:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941524.959927:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941524.959936:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962c00. 00010000:00000001:3.0:1773941524.959942:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941524.959946:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941524.959950:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941524.959953:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1e27 to hash 00000020:00000001:3.0:1773941524.959956:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941524.959957:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941524.959962:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941524.959966:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941524.959969:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941524.959974:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e27 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941524.959986:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e27 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941524.960018:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e27 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941524.960025:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941524.960028:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941524.960034:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941524.960036:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941524.960039:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941524.960041:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941524.960046:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941524.960051:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941524.960054:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941524.960056:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941524.960068:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241165696/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941524.960079:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941524.960082:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941524.960086:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941524.960089:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941524.960092:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941524.960095:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241165696:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941524.960100:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941524.960105:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941524.960108:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941524.960111:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941524.960116:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941524.960118:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941524.960127:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941524.960133:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941524.960142:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941524.960152:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241165696, portal 25 00000100:00000001:3.0:1773941524.960156:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941524.960169:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941524.960179:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241165696/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941540 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941524.960192:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941524.960197:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941524.960202:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a220. 00000100:00000200:3.0:1773941524.960208:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241165696, offset 0 00000400:00000200:3.0:1773941524.960218:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941524.960231:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941524.960243:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941524.960251:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941524.960257:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941524.960262:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941524.960265:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941524.960269:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[168, 167] 00000400:00000200:3.0:1773941524.960274:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941524.960279:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [274:274:256:4294967295] 192.168.204.153@tcp LPNI seq info [169:169:8:4294967295] 00000400:00000200:3.0:1773941524.960284:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941524.960319:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941524.960326:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941524.960333:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69600. 00000800:00000200:3.0:1773941524.960341:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941524.960349:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941524.960355:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941524.960374:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941524.960379:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941524.960382:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294965323 00000100:00000001:3.0:1773941524.960390:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294964302 00000100:00000001:3.0:1773941524.960395:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294965323 00000100:00000001:3.0:1773941524.960399:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294964302 00000100:00000001:3.0:1773941524.960410:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941524.960413:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941524.960416:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941524.960418:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941524.960422:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941524.960425:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941524.960429:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941524.960568:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941524.960579:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69600. 00000400:00000200:2.0:1773941524.960587:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941524.960597:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941524.960606:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a220 00000400:00000010:2.0:1773941524.960610:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a220. 00000100:00000001:2.0:1773941524.960619:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941524.960628:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241165696/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941540 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941524.960647:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941524.960654:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241165696/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941540 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941524.960690:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941524.960695:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941524.962409:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941524.962424:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941524.962427:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941524.962430:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941524.962436:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941524.962442:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941524.962449:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941524.962453:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941524.962458:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941524.962463:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608580 00000400:00000200:2.0:1773941524.962469:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941524.962474:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x12ed [1] + 224 00000800:00000001:2.0:1773941524.962481:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941524.962489:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941524.962492:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941524.962496:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941524.962502:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941524.962508:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241165696/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941540 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941524.962521:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241165696/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941540 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941524.962540:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941524.962549:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941524.962552:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941524.962592:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941524.962600:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941524.962603:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941524.962606:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941524.962612:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941524.962614:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941524.962617:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941524.962625:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241165696/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941540 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941524.962639:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241165696/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941540 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941524.962648:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941524.962650:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941524.962653:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941524.962655:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941524.962658:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941524.962661:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941524.962663:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941524.962670:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=103 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941524.962674:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941524.962677:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941524.962681:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941524.962684:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941524.962688:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941524.962691:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941524.962696:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941524.962701:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941524.962737:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941524.962740:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941524.962742:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941524.962744:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941524.962749:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241165696/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941540 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941524.962759:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941524.962760:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941524.962763:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941524.962766:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941524.962768:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941524.962770:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941524.962773:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941524.962774:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941524.962775:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941524.962777:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941524.962779:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941524.962782:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941524.962784:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941524.962787:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941524.962790:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 294 00000400:00000200:3.0:1773941524.962795:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941524.962800:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941524.962804:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941524.962807:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941524.962810:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941524.962812:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941524.962815:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941524.962818:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941524.962820:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941524.962823:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941524.962825:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941524.962828:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941524.962839:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941524.962841:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941524.962843:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941524.962846:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941524.962849:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941524.962852:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941524.962855:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941524.962858:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941524.962861:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941524.962864:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941524.962866:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941524.962868:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941524.962873:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90276d80. 00000020:00000040:3.0:1773941524.962875:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941524.962879:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941524.962881:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941524.962884:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941524.962886:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941524.962889:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941524.962894:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90262000. 00000100:00000001:3.0:1773941524.962898:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941524.962904:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90276d80 x1860112241165824/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941524.962913:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941524.962917:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941524.962922:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b11fae2f] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90276d80 x1860112241165824/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941524.962946:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941524.962949:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941524.962951:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941524.962954:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941524.962957:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941524.962965:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941524.962967:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941524.962970:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941524.962972:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941524.962978:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241165696/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941540 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941524.962986:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941524.962990:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941524.962993:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941524.962995:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941524.963001:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241165696/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941540 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941524.963010:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241165696:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941524.963016:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941524.963019:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941524.963023:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941524.963023:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941524.963025:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941524.963034:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1773941524.963035:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241165696/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941540 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941524.963042:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1773941524.963050:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941524.963054:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941524.963061:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941524.963063:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:1.0:1773941524.963063:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90276d80 x1860112241165824/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941524.963067:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:3.0:1773941524.963070:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941524.963072:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941524.963074:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 00000020:00000001:3.0:1773941524.963077:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941524.963083:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00010000:3.0:1773941524.963094:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e27 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941524.963105:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941524.963112:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941524.963122:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:3.0:1773941524.963125:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:1.0:1773941524.963129:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00010000:3.0:1773941524.963133:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e27 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:1.0:1773941524.963138:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941524.963151:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90276d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241165824:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:3.0:1773941524.963159:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:1.0:1773941524.963161:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941524.963167:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00010000:3.0:1773941524.963169:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e27 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:1.0:1773941524.963170:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941524.963176:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941524.963182:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0736c00. 02000000:00000001:1.0:1773941524.963188:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941524.963194:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:1.0:1773941524.963197:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941524.963201:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:1.0:1773941524.963204:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00010000:00010000:3.0:1773941524.963210:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e27 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:1.0:1773941524.963212:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941524.963219:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241165824, portal 25 00000100:00000001:1.0:1773941524.963227:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941524.963230:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109628800 : -128249599922816 : ffff8b5b90276d80) 00010000:00000001:3.0:1773941524.963240:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000040:1.0:1773941524.963242:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90276d80 x1860112241165824/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941534 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941524.963248:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941524.963258:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000001:1.0:1773941524.963261:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:3.0:1773941524.963267:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e27 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:1.0:1773941524.963267:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941524.963274:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1c38. 00000100:00000200:1.0:1773941524.963281:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241165824, offset 0 00010000:00010000:3.0:1773941524.963289:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:1.0:1773941524.963314:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 10000000:00000001:3.0:1773941524.963316:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941524.963324:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e27 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941524.963330:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941524.963342:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:1.0:1773941524.963343:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:3.0:1773941524.963347:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:1.0:1773941524.963351:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1773941524.963353:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941524.963358:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000400:00000200:1.0:1773941524.963359:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000020:00000001:3.0:1773941524.963365:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000200:1.0:1773941524.963368:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:3.0:1773941524.963373:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000200:1.0:1773941524.963376:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:3.0:1773941524.963388:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:1.0:1773941524.963395:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[168, 169] 00010000:00010000:3.0:1773941524.963396:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e27 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941524.963412:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 10000000:00000001:3.0:1773941524.963419:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:1.0:1773941524.963422:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [275:275:256:4294967295] 192.168.204.143@tcp LPNI seq info [170:170:8:4294967295] 10000000:00010000:3.0:1773941524.963425:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e27 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941524.963456:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:01000000:3.0:1773941524.963465:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:3.0:1773941524.963475:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941524.963481:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941524.963482:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00010000:3.0:1773941524.963487:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941524.963492:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000800:00000200:1.0:1773941524.963499:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941524.963500:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941524.963507:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000800:00000010:1.0:1773941524.963509:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b900. 00010000:00000001:3.0:1773941524.963510:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941524.963516:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941524.963522:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1e27 from hash 00000800:00000200:1.0:1773941524.963528:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:3.0:1773941524.963529:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941524.963535:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941524.963539:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941524.963541:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:3.0:1773941524.963546:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000800:00000200:1.0:1773941524.963547:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:3.0:1773941524.963567:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000200:1.0:1773941524.963571:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b900 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:3.0:1773941524.963572:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941524.963577:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941524.963581:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941524.963583:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941524.963586:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e27 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941524.963592:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:1.0:1773941524.963594:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941524.963595:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941524.963599:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941524.963602:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000400:00000200:1.0:1773941524.963604:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941524.963609:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294965326 00010000:00000001:3.0:1773941524.963615:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:1.0:1773941524.963616:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294965323 00010000:00000001:3.0:1773941524.963619:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941524.963623:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:1.0:1773941524.963624:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294965326 00010000:00010000:3.0:1773941524.963628:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e27 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941524.963631:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294965323 00000100:00000001:1.0:1773941524.963639:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941524.963646:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:3.0:1773941524.963647:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962c00 count: 0 00000100:00000001:1.0:1773941524.963650:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941524.963656:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941524.963665:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:3.0:1773941524.963667:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:1.0:1773941524.963668:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:3.0:1773941524.963671:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941524.963674:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941524.963679:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941524.963680:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941524.963687:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941524.963693:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241165696/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941540 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941524.963694:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941524.963709:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941524.963713:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941524.963717:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941524.963721:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941524.963724:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941524.963727:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941524.963729:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000800:00000200:2.0:1773941524.963779:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 02000000:00000010:3.0:1773941524.963784:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5ba0ab8200. 00000800:00000010:2.0:1773941524.963787:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b900. 02000000:00000001:3.0:1773941524.963792:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941524.963796:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000400:00000200:2.0:1773941524.963797:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000010:3.0:1773941524.963799:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:3.0:1773941524.963807:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000400:00000200:2.0:1773941524.963811:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:3.0:1773941524.963812:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941524.963819:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941524.963823:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1c38 10000000:01000000:3.0:1773941524.963824:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000400:00000010:2.0:1773941524.963838:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1c38. 00000100:00000001:2.0:1773941524.963845:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941524.963852:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90276d80 x1860112241165824/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941534 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941524.963863:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941524.963867:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90276d80 x1860112241165824/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941534 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941524.963876:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941524.963879:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000001:3.0:1773941524.963889:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941524.963896:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068600. 00000020:00000010:3.0:1773941524.963902:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab8200. 00000020:00000010:3.0:1773941524.963908:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab8600. 00000040:00000010:3.0:1773941524.963912:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab8e00. 00000020:00000010:3.0:1773941524.963917:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941524.963922:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab8a00. 80000000:00000010:3.0:1773941524.963927:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941524.963939:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab8000. 20000000:00000010:3.0:1773941524.963944:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5ba0ab9a00. 10000000:00000040:3.0:1773941524.963949:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:3.0:1773941524.963952:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941524.963957:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab8600. 00000040:00000010:3.0:1773941524.963962:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab8e00. 00000020:00000010:3.0:1773941524.963965:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941524.963969:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab8a00. 80000000:00000010:3.0:1773941524.963972:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941524.963976:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab8000. 20000000:00000010:3.0:1773941524.963980:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5ba0ab9a00. 00000020:00000010:3.0:1773941524.963983:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab8200. 10000000:00000010:3.0:1773941524.963986:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068600. 10000000:01000000:3.0:1773941524.963989:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941524.963992:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941524.963995:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941524.963997:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941524.963999:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941524.964000:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941524.964003:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941524.965550:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941524.965562:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941524.965565:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941524.965568:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941524.965573:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941524.965579:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941524.965586:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941524.965592:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941524.965598:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941524.965605:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608600 00000400:00000200:2.0:1773941524.965612:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941524.965620:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x12fd [1] + 0 00000800:00000001:2.0:1773941524.965629:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941524.965641:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941524.965645:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941524.965652:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941524.965658:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941524.965667:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90276d80 x1860112241165824/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941534 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941524.965687:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90276d80 x1860112241165824/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941534 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941524.965761:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941524.965788:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000100:00000001:1.0:1773941524.965791:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000001:2.0:1773941524.965794:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941524.965796:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941524.965806:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941524.965808:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941524.965811:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941524.965819:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941524.965832:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941524.965836:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941524.965844:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90276d80 x1860112241165824/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941534 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941524.965857:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90276d80 x1860112241165824/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941534 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941524.965866:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941524.965867:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941524.965870:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941524.965872:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941524.965874:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941524.965876:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941524.965878:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941524.965882:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=103 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941524.965886:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941524.965889:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941524.965891:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941524.965893:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941524.965896:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941524.965898:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941524.965903:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90276d80 x1860112241165824/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941534 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941524.965911:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941524.965914:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941524.965915:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941524.965918:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1773941524.965923:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:1.0:1773941524.965925:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941524.965927:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941524.965929:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941524.965931:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941524.965934:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941524.965938:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:1.0:1773941524.965943:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941524 v=0 (0 0 0 0) 00000100:00001000:1.0:1773941524.965957:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:1.0:1773941524.965966:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:1.0:1773941524.965969:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941524.965972:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:1.0:1773941524.965975:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941524.965978:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941524.965984:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:1.0:1773941524.965988:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:1.0:1773941524.965992:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:1.0:1773941524.965998:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:1.0:1773941524.966003:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:1.0:1773941524.966007:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941524.966452:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941524.966458:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941524.966468:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90276d80 x1860112241165824/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941534 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941524.966481:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90276d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241165824:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941524.966489:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941524.966492:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941524.966497:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90276d80 x1860112241165824/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941534 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941524.966507:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1773941524.966508:0:12246:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 02000000:00000001:0.0:1773941524.966509:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941524.966515:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0736c00. 02000000:00000001:0.0:1773941524.966520:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:1.0:1773941524.966523:0:12246:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000020:00000001:0.0:1773941524.966524:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941524.966529:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00100000:1.0:1773941524.966533:0:12246:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941524.966536:0:12246:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941524.966538:0:12246:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941524.966542:0:12246:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:0.0:1773941524.966546:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941524.966547:0:12246:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:0.0:1773941524.966551:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90262000. 00000001:00000001:1.0:1773941524.966554:0:12246:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941524.966557:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:1.0:1773941524.966559:0:12246:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:0.0:1773941524.966561:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:1.0:1773941524.966563:0:12246:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000010:0.0:1773941524.966566:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90276d80. 00000001:00000001:1.0:1773941524.966568:0:12246:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941524.966575:0:12246:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941524.966575:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941524.966578:0:12246:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941524.966580:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941524.966585:0:12246:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941524.966586:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941524.966595:0:12246:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941524.966597:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941524.966599:0:12246:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941524.966602:0:12246:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941524.966605:0:12246:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941524.966607:0:12246:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:0.0:1773941524.966612:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941524.966613:0:12246:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941524.966620:0:12246:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:0.0:1773941524.966621:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941524.966626:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941524.966648:0:12246:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941524.966652:0:12246:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941524.966663:0:12246:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941524.966667:0:12246:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941524.966671:0:12246:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941524.966675:0:12246:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941524.966677:0:12246:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941524.966680:0:12246:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941524.966683:0:12246:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941524.968792:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962c00. 00010000:00000010:3.1:1773941524.968803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941525.343862:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941525.343870:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941525.343881:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941526.047837:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941526.047845:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941526.047850:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941526.047852:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941526.047854:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941526.047855:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941526.047858:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941526.047860:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941526.047862:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941526.047865:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941526.047867:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941526.047879:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941526.047925:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941526.047930:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941526.047933:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941526.047939:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941526.047946:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5ba0ab9a00. 00000100:00000001:3.0:1773941526.047953:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941526.047957:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941526.047959:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941526.047967:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962c00. 00010000:00000001:3.0:1773941526.047972:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941526.047975:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941526.047995:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941526.047997:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1e2e to hash 00000020:00000001:3.0:1773941526.047999:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941526.048000:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941526.048003:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941526.048006:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941526.048009:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941526.048013:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e2e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941526.048022:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e2e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941526.048037:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e2e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941526.048042:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941526.048044:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941526.048050:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941526.048051:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941526.048053:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941526.048055:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941526.048059:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941526.048067:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941526.048069:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941526.048071:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941526.048097:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241165952/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941526.048106:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941526.048109:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941526.048112:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941526.048115:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941526.048117:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941526.048120:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241165952:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941526.048124:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941526.048129:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941526.048132:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941526.048134:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941526.048138:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941526.048140:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941526.048162:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941526.048165:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941526.048171:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941526.048176:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241165952, portal 25 00000100:00000001:3.0:1773941526.048177:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941526.048178:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941526.048182:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241165952/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941542 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941526.048187:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941526.048190:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941526.048191:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a4c8. 00000100:00000200:3.0:1773941526.048193:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241165952, offset 0 00000400:00000200:3.0:1773941526.048198:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941526.048204:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941526.048227:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941526.048232:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941526.048241:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941526.048245:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941526.048248:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941526.048251:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[170, 169] 00000400:00000200:3.0:1773941526.048255:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941526.048259:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [276:276:256:4294967295] 192.168.204.153@tcp LPNI seq info [171:171:8:4294967295] 00000400:00000200:3.0:1773941526.048264:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941526.048303:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941526.048310:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941526.048331:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69600. 00000800:00000200:3.0:1773941526.048336:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941526.048343:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941526.048347:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941526.048363:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941526.048366:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941526.048368:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294966411 00000100:00000001:3.0:1773941526.048374:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294965326 00000100:00000001:3.0:1773941526.048378:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294966411 00000100:00000001:3.0:1773941526.048380:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294965326 00000100:00000001:3.0:1773941526.048382:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941526.048384:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941526.048386:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941526.048387:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941526.048389:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941526.048392:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941526.048394:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941526.048492:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941526.048497:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69600. 00000400:00000200:2.0:1773941526.048501:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941526.048505:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941526.048510:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a4c8 00000400:00000010:2.0:1773941526.048512:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a4c8. 00000100:00000001:2.0:1773941526.048516:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941526.048520:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241165952/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941542 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941526.048528:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941526.048530:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241165952/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941542 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941526.048536:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941526.048537:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941526.049862:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941526.049874:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941526.049877:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941526.049880:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941526.049891:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941526.049896:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941526.049903:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941526.049907:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941526.049912:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941526.049916:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608680 00000400:00000200:2.0:1773941526.049920:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941526.049926:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x1305 [1] + 224 00000800:00000001:2.0:1773941526.049933:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941526.049941:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941526.049944:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941526.049948:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941526.049952:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941526.049964:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241165952/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941542 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941526.049975:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241165952/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941542 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941526.049990:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941526.049997:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941526.050000:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941526.050054:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941526.050060:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941526.050061:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941526.050063:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941526.050066:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941526.050074:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941526.050077:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941526.050082:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241165952/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941542 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941526.050090:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241165952/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941542 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941526.050095:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941526.050096:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941526.050098:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941526.050100:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941526.050102:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941526.050104:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941526.050105:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941526.050110:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=105 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941526.050113:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941526.050115:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941526.050118:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941526.050120:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941526.050122:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941526.050124:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941526.050127:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941526.050130:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941526.050133:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941526.050134:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941526.050136:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941526.050137:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941526.050140:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241165952/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941542 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941526.050146:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941526.050146:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941526.050147:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941526.050153:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941526.050155:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941526.050156:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941526.050157:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941526.050158:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941526.050159:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941526.050160:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941526.050161:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941526.050163:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941526.050164:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941526.050166:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941526.050168:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 295 00000400:00000200:3.0:1773941526.050171:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941526.050174:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941526.050177:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941526.050179:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941526.050181:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941526.050182:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941526.050185:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941526.050186:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941526.050188:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941526.050189:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941526.050191:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941526.050192:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941526.050194:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941526.050195:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941526.050196:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941526.050197:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941526.050200:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941526.050202:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941526.050203:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941526.050205:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941526.050207:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941526.050211:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941526.050213:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941526.050215:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941526.050218:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90274700. 00000020:00000040:3.0:1773941526.050219:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941526.050222:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941526.050223:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941526.050225:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941526.050226:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941526.050228:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941526.050231:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90262000. 00000100:00000001:3.0:1773941526.050233:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941526.050237:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90274700 x1860112241166080/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941526.050243:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941526.050245:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941526.050248:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2e4aa82] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90274700 x1860112241166080/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941526.050268:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941526.050271:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941526.050282:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941526.050285:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941526.050288:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941526.050290:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941526.050292:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941526.050295:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941526.050298:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941526.050304:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1773941526.050305:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241165952/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941542 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941526.050308:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941526.050311:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941526.050318:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90274700 x1860112241166080/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941526.050320:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941526.050328:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941526.050333:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941526.050335:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941526.050336:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941526.050339:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1773941526.050340:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941526.050342:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941526.050343:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941526.050346:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90274700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241166080:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941526.050351:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941526.050353:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941526.050355:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941526.050358:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241165952/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941542 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1773941526.050358:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941526.050362:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0c00. 02000000:00000001:0.0:1773941526.050366:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941526.050373:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941526.050378:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000100:00100000:3.0:1773941526.050381:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241165952:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000400:00000010:0.0:1773941526.050383:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941526.050388:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241166080, portal 25 00000100:00000001:3.0:1773941526.050390:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941526.050391:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941526.050393:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109618944 : -128249599932672 : ffff8b5b90274700) 00000100:00000001:3.0:1773941526.050395:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941526.050398:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941526.050399:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941526.050404:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241165952/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941542 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941526.050407:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90274700 x1860112241166080/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941536 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941526.050419:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941526.050422:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:3.0:1773941526.050423:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000040:0.0:1773941526.050426:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000100:00000001:3.0:1773941526.050429:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000400:00000010:0.0:1773941526.050430:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6cee0. 00000100:00000001:3.0:1773941526.050431:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000200:0.0:1773941526.050434:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241166080, offset 0 00010000:00000001:3.0:1773941526.050435:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:3.0:1773941526.050440:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:0.0:1773941526.050441:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000020:00000001:3.0:1773941526.050442:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941526.050445:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 00000400:00000200:0.0:1773941526.050449:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:3.0:1773941526.050450:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000200:0.0:1773941526.050454:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941526.050457:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:3.0:1773941526.050460:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000200:0.0:1773941526.050461:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941526.050465:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:3.0:1773941526.050467:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e2e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941526.050468:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941526.050475:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[170, 171] 00010000:00000001:3.0:1773941526.050484:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:0.0:1773941526.050485:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941526.050491:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [277:277:256:4294967295] 192.168.204.143@tcp LPNI seq info [172:172:8:4294967295] 00010000:00010000:3.0:1773941526.050492:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e2e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941526.050498:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941526.050504:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:0.0:1773941526.050508:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00010000:3.0:1773941526.050509:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e2e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941526.050518:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941526.050522:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b91e1e300. 00000800:00000200:0.0:1773941526.050528:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:3.0:1773941526.050536:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:0.0:1773941526.050536:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:3.0:1773941526.050539:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:0.0:1773941526.050541:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b91e1e300 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:3.0:1773941526.050543:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e2e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941526.050549:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941526.050551:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941526.050553:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941526.050554:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:3.0:1773941526.050558:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e2e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941526.050559:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941526.050564:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294966413 00010000:00010000:3.0:1773941526.050567:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 00000100:00000001:0.0:1773941526.050569:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294966411 10000000:00000001:3.0:1773941526.050570:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941526.050574:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e2e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941526.050574:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294966413 00000100:00000001:0.0:1773941526.050580:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294966411 00010000:00000001:3.0:1773941526.050582:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:3.0:1773941526.050585:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941526.050586:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941526.050587:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000020:00000001:3.0:1773941526.050592:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000100:00000001:0.0:1773941526.050594:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1773941526.050596:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000100:00000001:0.0:1773941526.050599:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941526.050600:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000001:0.0:1773941526.050602:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941526.050604:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e2e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941526.050606:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1773941526.050611:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 10000000:00000001:3.0:1773941526.050612:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:0.0:1773941526.050614:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000800:00000010:2.0:1773941526.050615:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b91e1e300. 10000000:00010000:3.0:1773941526.050616:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e2e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941526.050617:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:2.0:1773941526.050621:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1773941526.050622:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941526.050625:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941526.050628:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:3.0:1773941526.050631:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941526.050632:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:0.0:1773941526.050632:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941526.050639:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6cee0 10000000:00000040:3.0:1773941526.050640:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941526.050644:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1773941526.050644:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6cee0. 00000100:00000001:2.0:1773941526.050650:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00010000:3.0:1773941526.050651:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941526.050654:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000200:2.0:1773941526.050656:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90274700 x1860112241166080/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941536 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941526.050663:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941526.050665:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90274700 x1860112241166080/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941536 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941526.050669:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941526.050670:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1773941526.050766:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941526.050769:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941526.050770:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941526.050772:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941526.050775:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1e2e from hash 00010000:00000001:3.0:1773941526.050778:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941526.050779:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941526.050780:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941526.050781:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:3.0:1773941526.050783:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941526.050785:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:3.0:1773941526.050789:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941526.050790:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941526.050792:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941526.050794:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941526.050799:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e2e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941526.050807:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941526.050809:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941526.050811:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941526.050813:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941526.050815:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941526.050816:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941526.050818:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941526.050822:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e2e lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:3.0:1773941526.050832:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962c00 count: 0 00010000:00000001:3.0:1773941526.050844:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941526.050847:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941526.050851:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941526.050866:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241165952/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941542 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941526.050876:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941526.050879:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941526.050883:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941526.050887:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941526.050889:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941526.050892:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941526.050906:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941526.050925:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5ba0ab9a00. 02000000:00000001:3.0:1773941526.050928:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941526.050929:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941526.050930:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:3.0:1773941526.050933:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941526.050934:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941526.050935:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941526.050937:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941526.050939:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941526.050942:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068600. 00000020:00000010:3.0:1773941526.050945:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab9a00. 00000020:00000010:3.0:1773941526.050948:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab8000. 00000040:00000010:3.0:1773941526.050951:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab8a00. 00000020:00000010:3.0:1773941526.050954:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941526.050958:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab8e00. 80000000:00000010:3.0:1773941526.050961:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941526.050964:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab8600. 20000000:00000010:3.0:1773941526.050968:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5ba0ab9600. 10000000:00000040:3.0:1773941526.050971:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:3.0:1773941526.050973:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941526.050976:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab8000. 00000040:00000010:3.0:1773941526.050978:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab8a00. 00000020:00000010:3.0:1773941526.050980:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941526.050983:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab8e00. 80000000:00000010:3.0:1773941526.050985:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941526.050996:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab8600. 20000000:00000010:3.0:1773941526.050999:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5ba0ab9600. 00000020:00000010:3.0:1773941526.051003:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab9a00. 10000000:00000010:3.0:1773941526.051005:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068600. 10000000:01000000:3.0:1773941526.051008:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941526.051011:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941526.051013:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941526.051015:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941526.051016:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941526.051018:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941526.051020:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941526.052112:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941526.052121:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941526.052124:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941526.052126:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941526.052131:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941526.052152:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941526.052157:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941526.052159:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941526.052162:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941526.052164:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608700 00000400:00000200:2.0:1773941526.052167:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941526.052175:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1315 [1] + 0 00000800:00000001:2.0:1773941526.052180:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941526.052184:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941526.052186:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941526.052189:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941526.052191:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941526.052195:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90274700 x1860112241166080/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941536 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941526.052203:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90274700 x1860112241166080/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941536 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941526.052236:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941526.052241:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941526.052244:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941526.052269:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941526.052290:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941526.052294:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941526.052295:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941526.052298:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941526.052302:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941526.052304:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941526.052306:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941526.052312:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90274700 x1860112241166080/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941536 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941526.052337:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90274700 x1860112241166080/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941536 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941526.052342:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941526.052343:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941526.052345:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941526.052346:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941526.052347:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941526.052348:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941526.052350:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941526.052352:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=105 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941526.052355:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941526.052356:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941526.052357:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941526.052359:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941526.052360:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941526.052362:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941526.052371:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90274700 x1860112241166080/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941536 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941526.052377:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941526.052378:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941526.052379:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941526.052381:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941526.052384:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941526.052386:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941526.052387:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941526.052388:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941526.052389:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941526.052391:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941526.052392:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941526.052395:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941526 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941526.052397:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941526.052401:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941526.052403:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941526.052404:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941526.052406:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941526.052407:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941526.052409:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941526.052411:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941526.052413:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941526.052415:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941526.052417:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941526.052419:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941526.052742:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941526.052747:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941526.052755:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90274700 x1860112241166080/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941536 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941526.052766:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90274700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241166080:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941526.052768:0:12247:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:1.0:1773941526.052772:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941526.052776:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1773941526.052782:0:12247:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000040:1.0:1773941526.052783:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90274700 x1860112241166080/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941536 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941526.052788:0:12247:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941526.052793:0:12247:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941526.052796:0:12247:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:1.0:1773941526.052799:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941526.052801:0:12247:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:1.0:1773941526.052803:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941526.052806:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0c00. 02000000:00000001:1.0:1773941526.052809:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941526.052812:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941526.052812:0:12247:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:1.0:1773941526.052817:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941526.052818:0:12247:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941526.052820:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:0.0:1773941526.052823:0:12247:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:1.0:1773941526.052824:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90262000. 00000001:00000001:0.0:1773941526.052826:0:12247:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:1.0:1773941526.052829:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:0.0:1773941526.052830:0:12247:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941526.052833:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:00000001:0.0:1773941526.052834:0:12247:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941526.052837:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90274700. 00000100:00000001:0.0:1773941526.052838:0:12247:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941526.052845:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:0.0:1773941526.052846:0:12247:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941526.052848:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941526.052850:0:12247:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941526.052852:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941526.052854:0:12247:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941526.052858:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941526.052860:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941526.052862:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941526.052882:0:12247:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941526.052886:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941526.052888:0:12247:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941526.052891:0:12247:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941526.052893:0:12247:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941526.052894:0:12247:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941526.052908:0:12247:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941526.052910:0:12247:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941526.052911:0:12247:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941526.052914:0:12247:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941526.052916:0:12247:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941526.052918:0:12247:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941526.052919:0:12247:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941526.052921:0:12247:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941526.052922:0:12247:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941526.056742:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962c00. 00010000:00000010:3.1:1773941526.056750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941526.367890:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941526.367896:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941526.367899:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941527.391840:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941527.391845:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941527.391849:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941527.583805:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941527.583813:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941527.583818:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941527.583820:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941527.583821:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941527.583823:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941527.583825:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941527.583827:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941527.583829:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941527.583832:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941527.583833:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941527.583844:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941527.583848:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941527.583852:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941527.583855:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941527.583861:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941527.583868:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5ba0ab8600. 00000100:00000001:3.0:1773941527.583891:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941527.583895:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941527.583912:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941527.583919:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962c00. 00010000:00000001:3.0:1773941527.583924:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941527.583927:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941527.583943:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941527.583945:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1e35 to hash 00000020:00000001:3.0:1773941527.583947:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941527.583948:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941527.583951:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941527.583970:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941527.583973:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941527.583976:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e35 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941527.583985:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e35 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941527.584011:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e35 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941527.584016:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941527.584018:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941527.584023:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941527.584025:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941527.584027:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941527.584028:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941527.584032:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941527.584036:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941527.584037:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941527.584039:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941527.584048:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241166208/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941527.584056:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941527.584061:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941527.584064:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941527.584067:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941527.584068:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941527.584071:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241166208:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941527.584075:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941527.584079:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941527.584081:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941527.584084:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941527.584088:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941527.584089:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941527.584095:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941527.584099:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941527.584104:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941527.584111:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241166208, portal 25 00000100:00000001:3.0:1773941527.584122:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941527.584124:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941527.584128:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241166208/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941543 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941527.584134:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941527.584137:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941527.584138:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6ad48. 00000100:00000200:3.0:1773941527.584141:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241166208, offset 0 00000400:00000200:3.0:1773941527.584145:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941527.584152:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941527.584159:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941527.584163:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941527.584167:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941527.584171:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941527.584173:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941527.584175:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[172, 171] 00000400:00000200:3.0:1773941527.584188:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941527.584191:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [278:278:256:4294967295] 192.168.204.153@tcp LPNI seq info [173:173:8:4294967295] 00000400:00000200:3.0:1773941527.584199:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941527.584207:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941527.584212:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941527.584216:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69c00. 00000800:00000200:3.0:1773941527.584222:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941527.584228:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941527.584233:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69c00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941527.584304:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941527.584310:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941527.584314:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294967947 00000100:00000001:3.0:1773941527.584321:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294966413 00000100:00000001:3.0:1773941527.584327:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294967947 00000100:00000001:3.0:1773941527.584330:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294966413 00000100:00000001:3.0:1773941527.584333:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941527.584337:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941527.584340:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941527.584341:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941527.584344:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941527.584346:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941527.584348:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941527.584400:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941527.584405:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69c00. 00000400:00000200:2.0:1773941527.584410:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941527.584414:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941527.584418:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6ad48 00000400:00000010:2.0:1773941527.584420:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6ad48. 00000100:00000001:2.0:1773941527.584423:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941527.584427:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241166208/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941543 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941527.584435:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941527.584438:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241166208/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941543 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941527.584443:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941527.584444:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941527.585475:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941527.585485:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941527.585488:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941527.585489:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941527.585494:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941527.585504:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941527.585509:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941527.585512:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941527.585515:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941527.585518:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608780 00000400:00000200:2.0:1773941527.585521:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941527.585524:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x131d [1] + 224 00000800:00000001:2.0:1773941527.585530:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941527.585537:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941527.585539:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941527.585541:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941527.585545:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941527.585549:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241166208/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941543 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941527.585556:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241166208/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941543 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941527.585571:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941527.585578:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941527.585580:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941527.585666:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941527.585671:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941527.585672:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941527.585674:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941527.585677:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941527.585678:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941527.585680:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941527.585684:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241166208/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941543 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941527.585691:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241166208/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941543 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941527.585720:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941527.585722:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941527.585723:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941527.585725:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941527.585727:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941527.585729:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941527.585731:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941527.585736:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=106 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941527.585738:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941527.585741:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941527.585744:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941527.585746:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941527.585748:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941527.585751:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941527.585754:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941527.585757:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941527.585759:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941527.585761:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941527.585762:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941527.585763:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941527.585766:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241166208/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941543 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941527.585770:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941527.585771:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941527.585772:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941527.585774:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941527.585776:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941527.585777:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941527.585778:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941527.585779:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941527.585780:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941527.585781:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941527.585786:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941527.585788:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941527.585789:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941527.585791:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941527.585793:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 296 00000400:00000200:3.0:1773941527.585797:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941527.585799:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941527.585802:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941527.585804:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941527.585806:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941527.585808:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941527.585810:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941527.585811:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941527.585813:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941527.585815:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941527.585816:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941527.585818:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941527.585819:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941527.585820:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941527.585822:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941527.585823:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941527.585825:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941527.585827:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941527.585829:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941527.585831:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941527.585833:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941527.585835:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941527.585837:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941527.585838:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941527.585841:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277b80. 00000020:00000040:3.0:1773941527.585843:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941527.585845:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941527.585849:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941527.585850:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941527.585852:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941527.585853:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941527.585856:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90263000. 00000100:00000001:3.0:1773941527.585859:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941527.585863:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90277b80 x1860112241166336/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941527.585868:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941527.585870:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941527.585873:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000010dacf3] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90277b80 x1860112241166336/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941527.585888:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941527.585890:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941527.585891:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941527.585893:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941527.585894:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941527.585895:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941527.585897:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941527.585898:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941527.585900:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941527.585903:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241166208/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941543 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941527.585908:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941527.585909:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941527.585911:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941527.585912:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941527.585915:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241166208/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941543 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941527.585921:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241166208:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941527.585925:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941527.585932:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941527.585934:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941527.585936:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941527.585937:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1773941527.585939:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241166208/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941543 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941527.585943:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941527.585947:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941527.585948:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000010:3.0:1773941527.585950:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941527.585953:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941527.585954:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:3.0:1773941527.585957:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000040:1.0:1773941527.585959:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90277b80 x1860112241166336/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941527.585960:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941527.585963:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941527.585965:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 00000020:00000001:3.0:1773941527.585968:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941527.585972:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00010000:3.0:1773941527.585977:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e35 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941527.585984:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:3.0:1773941527.585987:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941527.585988:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941527.585991:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e35 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:1.0:1773941527.585994:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:3.0:1773941527.585999:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000001:1.0:1773941527.586000:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00010000:3.0:1773941527.586002:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e35 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:1.0:1773941527.586006:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941527.586010:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941527.586012:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00100000:1.0:1773941527.586015:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241166336:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00010000:3.0:1773941527.586016:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e35 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941527.586027:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:3.0:1773941527.586029:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 02000000:00000001:1.0:1773941527.586032:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:3.0:1773941527.586034:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941527.586036:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941527.586038:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 02000000:00000001:1.0:1773941527.586041:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00010000:3.0:1773941527.586042:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e35 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:1.0:1773941527.586050:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0737c00. 00010000:00010000:3.0:1773941527.586051:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:3.0:1773941527.586055:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 02000000:00000001:1.0:1773941527.586057:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 10000000:00010000:3.0:1773941527.586058:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e35 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941527.586065:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:1.0:1773941527.586066:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941527.586067:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941527.586070:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941527.586072:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000400:00000010:1.0:1773941527.586073:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000020:00000001:3.0:1773941527.586074:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941527.586079:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941527.586082:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000010:1.0:1773941527.586083:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00010000:00010000:3.0:1773941527.586084:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e35 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:1.0:1773941527.586090:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241166336, portal 25 10000000:00000001:3.0:1773941527.586091:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941527.586094:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e35 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941527.586095:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941527.586099:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109632384 : -128249599919232 : ffff8b5b90277b80) 10000000:01000000:3.0:1773941527.586102:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:3.0:1773941527.586105:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941527.586107:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941527.586109:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90277b80 x1860112241166336/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941537 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1773941527.586110:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941527.586113:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:3.0:1773941527.586116:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941527.586118:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941527.586121:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941527.586123:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941527.586127:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:3.0:1773941527.586128:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1e35 from hash 00000100:00000040:1.0:1773941527.586134:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941527.586135:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941527.586138:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941527.586140:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941527.586141:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:3.0:1773941527.586142:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000010:1.0:1773941527.586142:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1440. 00010000:00000001:3.0:1773941527.586145:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:3.0:1773941527.586148:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941527.586149:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000200:1.0:1773941527.586149:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241166336, offset 0 00010000:00000001:3.0:1773941527.586151:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941527.586153:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941527.586157:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e35 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941527.586162:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941527.586165:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941527.586168:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941527.586169:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941527.586171:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000400:00000200:1.0:1773941527.586172:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941527.586173:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941527.586175:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941527.586178:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941527.586180:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e35 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941527.586183:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000040:3.0:1773941527.586187:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962c00 count: 0 00010000:00000001:3.0:1773941527.586195:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:1.0:1773941527.586197:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:3.0:1773941527.586198:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941527.586202:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1773941527.586205:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000100:00000040:3.0:1773941527.586208:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241166208/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941543 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941527.586213:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941527.586220:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1773941527.586222:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941527.586225:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:1.0:1773941527.586227:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[172, 173] 02000000:00000010:3.0:1773941527.586228:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941527.586232:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941527.586235:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:1.0:1773941527.586236:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000020:00000040:3.0:1773941527.586237:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941527.586241:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000200:1.0:1773941527.586257:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [279:279:256:4294967295] 192.168.204.143@tcp LPNI seq info [174:174:8:4294967295] 02000000:00000010:3.0:1773941527.586265:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5ba0ab8600. 00000400:00000200:1.0:1773941527.586266:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 02000000:00000001:3.0:1773941527.586269:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941527.586272:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941527.586274:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:3.0:1773941527.586277:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941527.586280:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941527.586284:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:1.0:1773941527.586285:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 10000000:01000000:3.0:1773941527.586286:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941527.586290:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941527.586293:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068600. 00000800:00000200:1.0:1773941527.586296:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000020:00000010:3.0:1773941527.586297:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab8600. 00000800:00000010:1.0:1773941527.586304:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b200. 00000020:00000010:3.0:1773941527.586305:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab8e00. 00000040:00000010:3.0:1773941527.586310:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab8a00. 00000800:00000200:1.0:1773941527.586315:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000020:00000010:3.0:1773941527.586317:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941527.586322:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab8000. 80000000:00000010:3.0:1773941527.586327:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00000800:00000200:1.0:1773941527.586327:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00040000:00000010:3.0:1773941527.586334:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab9c00. 00000800:00000200:1.0:1773941527.586334:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b200 type 1, nob 616 niov 1 nkiov 1 20000000:00000010:3.0:1773941527.586339:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5ba0ab9200. 10000000:00000040:3.0:1773941527.586344:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:3.0:1773941527.586346:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941527.586349:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab8e00. 00000100:00000001:1.0:1773941527.586350:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:3.0:1773941527.586352:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab8a00. 00000020:00000010:3.0:1773941527.586357:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000400:00000200:1.0:1773941527.586360:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000010:3.0:1773941527.586362:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab8000. 00000100:00000001:1.0:1773941527.586364:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294967949 80000000:00000010:3.0:1773941527.586366:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941527.586373:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab9c00. 00000100:00000001:1.0:1773941527.586375:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294967947 20000000:00000010:3.0:1773941527.586376:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5ba0ab9200. 00000020:00000010:3.0:1773941527.586380:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab8600. 10000000:00000010:3.0:1773941527.586383:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068600. 00000100:00000001:1.0:1773941527.586385:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294967949 10000000:01000000:3.0:1773941527.586386:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941527.586389:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941527.586392:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941527.586394:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 00000100:00000001:1.0:1773941527.586394:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294967947 10000000:00000001:3.0:1773941527.586397:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 00000100:00000001:1.0:1773941527.586401:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:00000040:3.0:1773941527.586403:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 00000100:00000001:1.0:1773941527.586407:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941527.586408:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:1.0:1773941527.586413:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941527.586416:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941527.586420:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941527.586422:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1773941527.586424:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941527.586425:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941527.586426:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941527.586427:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941527.586434:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941527.586440:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b200. 00000400:00000200:2.0:1773941527.586446:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941527.586451:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941527.586455:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1440 00000400:00000010:2.0:1773941527.586457:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1440. 00000100:00000001:2.0:1773941527.586461:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941527.586466:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277b80 x1860112241166336/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941537 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941527.586475:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941527.586478:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90277b80 x1860112241166336/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941537 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941527.586498:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941527.586500:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941527.587663:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941527.587670:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941527.587671:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941527.587673:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941527.587676:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941527.587680:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941527.587684:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941527.587686:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941527.587689:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941527.587691:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608800 00000400:00000200:2.0:1773941527.587693:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941527.587696:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x132d [1] + 0 00000800:00000001:2.0:1773941527.587700:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941527.587741:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941527.587744:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941527.587748:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941527.587753:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941527.587757:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277b80 x1860112241166336/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941537 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941527.587767:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90277b80 x1860112241166336/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941537 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941527.587784:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941527.587789:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941527.587791:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941527.587838:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941527.587848:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941527.587852:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941527.587854:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941527.587856:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941527.587861:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941527.587863:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941527.587866:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941527.587872:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277b80 x1860112241166336/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941537 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941527.587884:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277b80 x1860112241166336/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941537 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941527.587892:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941527.587893:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941527.587896:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941527.587898:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941527.587899:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941527.587901:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941527.587903:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941527.587907:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=106 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941527.587910:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941527.587913:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941527.587915:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941527.587917:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941527.587919:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941527.587922:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941527.587927:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90277b80 x1860112241166336/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941537 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941527.587934:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941527.587937:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941527.587938:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941527.587941:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1773941527.587945:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:1.0:1773941527.587947:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941527.587949:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941527.587951:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941527.587958:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941527.587960:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941527.587963:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:1.0:1773941527.587967:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941527 v=0 (0 0 0 0) 00000100:00001000:1.0:1773941527.587970:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:1.0:1773941527.587976:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:1.0:1773941527.587979:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941527.587981:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:1.0:1773941527.587983:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941527.587985:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941527.587988:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:1.0:1773941527.587991:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:1.0:1773941527.587993:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:1.0:1773941527.587995:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:1.0:1773941527.587999:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:1.0:1773941527.588001:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941527.588208:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941527.588212:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941527.588217:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90277b80 x1860112241166336/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941537 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941527.588224:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241166336:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941527.588228:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941527.588230:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941527.588232:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90277b80 x1860112241166336/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941537 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941527.588234:0:12248:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:1.0:1773941527.588259:0:12248:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:0.0:1773941527.588264:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:1.0:1773941527.588265:0:12248:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:0.0:1773941527.588268:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1773941527.588270:0:12248:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:0.0:1773941527.588272:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0737c00. 10000000:01000000:1.0:1773941527.588274:0:12248:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:0.0:1773941527.588276:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941527.588278:0:12248:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:0.0:1773941527.588279:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941527.588282:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:1.0:1773941527.588284:0:12248:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:0.0:1773941527.588285:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941527.588288:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90263000. 02000000:00000001:0.0:1773941527.588290:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:1.0:1773941527.588291:0:12248:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941527.588292:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941527.588294:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277b80. 00000001:00000001:1.0:1773941527.588296:0:12248:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:0.0:1773941527.588298:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:1.0:1773941527.588300:0:12248:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:0.0:1773941527.588300:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941527.588305:0:12248:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941527.588305:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941527.588308:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000001:1.0:1773941527.588309:0:12248:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941527.588311:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941527.588313:0:12248:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941527.588314:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941527.588317:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941527.588320:0:12248:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941527.588325:0:12248:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941527.588327:0:12248:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941527.588329:0:12248:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941527.588332:0:12248:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941527.588341:0:12248:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941527.588342:0:12248:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941527.588344:0:12248:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941527.588359:0:12248:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941527.588361:0:12248:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941527.588363:0:12248:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941527.588367:0:12248:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941527.588370:0:12248:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941527.588373:0:12248:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941527.588375:0:12248:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941527.588378:0:12248:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941527.588380:0:12248:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941527.591736:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962c00. 00010000:00000010:3.1:1773941527.591743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941528.415911:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941528.415921:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941528.415946:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941529.119843:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941529.119852:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941529.119857:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941529.119860:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941529.119861:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941529.119863:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941529.119866:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941529.119868:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941529.119870:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941529.119873:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941529.119875:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941529.119903:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941529.119908:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941529.119912:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941529.119916:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941529.119921:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941529.119929:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5ba0ab8000. 00000100:00000001:3.0:1773941529.119935:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941529.119939:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941529.119941:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941529.119948:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962c00. 00010000:00000001:3.0:1773941529.119953:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941529.119973:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941529.119977:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941529.119980:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1e3c to hash 00000020:00000001:3.0:1773941529.119982:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941529.119984:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941529.119987:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941529.119990:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941529.119993:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941529.119997:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e3c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941529.120024:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e3c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941529.120036:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e3c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941529.120042:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941529.120044:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941529.120050:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941529.120052:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941529.120054:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941529.120056:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941529.120060:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941529.120065:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941529.120067:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941529.120069:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941529.120078:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241166464/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941529.120087:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941529.120089:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941529.120093:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941529.120095:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941529.120097:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941529.120101:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241166464:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941529.120104:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941529.120110:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941529.120112:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941529.120115:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941529.120120:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941529.120122:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941529.120129:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941529.120137:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941529.120144:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941529.120151:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241166464, portal 25 00000100:00000001:3.0:1773941529.120153:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941529.120154:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941529.120160:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241166464/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941545 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941529.120168:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941529.120171:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941529.120174:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6ad48. 00000100:00000200:3.0:1773941529.120177:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241166464, offset 0 00000400:00000200:3.0:1773941529.120182:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941529.120190:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941529.120198:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941529.120203:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941529.120207:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941529.120211:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941529.120213:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941529.120239:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[174, 173] 00000400:00000200:3.0:1773941529.120244:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941529.120265:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [280:280:256:4294967295] 192.168.204.153@tcp LPNI seq info [175:175:8:4294967295] 00000400:00000200:3.0:1773941529.120270:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941529.120279:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941529.120284:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941529.120290:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69700. 00000800:00000200:3.0:1773941529.120296:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941529.120303:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941529.120307:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69700 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941529.120326:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941529.120330:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941529.120333:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294969483 00000100:00000001:3.0:1773941529.120346:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294967949 00000100:00000001:3.0:1773941529.120351:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294969483 00000100:00000001:3.0:1773941529.120354:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294967949 00000100:00000001:3.0:1773941529.120357:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941529.120360:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941529.120362:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941529.120364:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941529.120367:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941529.120370:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941529.120373:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941529.120471:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941529.120477:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69700. 00000400:00000200:2.0:1773941529.120482:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941529.120488:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941529.120492:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6ad48 00000400:00000010:2.0:1773941529.120502:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6ad48. 00000100:00000001:2.0:1773941529.120507:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941529.120512:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241166464/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941545 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941529.120523:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941529.120526:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241166464/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941545 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941529.120532:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941529.120534:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941529.121407:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941529.121419:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941529.121422:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941529.121424:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941529.121429:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941529.121434:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941529.121440:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941529.121443:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941529.121447:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941529.121451:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608880 00000400:00000200:2.0:1773941529.121455:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941529.121459:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x1335 [1] + 224 00000800:00000001:2.0:1773941529.121465:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941529.121471:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941529.121473:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941529.121476:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941529.121481:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941529.121486:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241166464/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941545 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941529.121497:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241166464/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941545 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941529.121512:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941529.121529:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941529.121532:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941529.121535:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941529.121542:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941529.121545:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941529.121548:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941529.121552:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941529.121555:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941529.121557:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941529.121563:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241166464/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941545 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941529.121573:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241166464/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941545 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941529.121579:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941529.121581:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941529.121583:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941529.121586:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941529.121588:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941529.121590:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941529.121592:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941529.121596:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=108 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941529.121599:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941529.121601:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941529.121604:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941529.121607:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941529.121617:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941529.121620:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941529.121624:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941529.121628:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941529.121632:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941529.121634:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941529.121635:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941529.121637:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941529.121640:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241166464/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941545 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941529.121646:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941529.121647:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941529.121649:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941529.121651:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941529.121653:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941529.121654:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941529.121656:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941529.121657:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941529.121658:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941529.121659:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941529.121660:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941529.121662:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941529.121664:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941529.121666:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941529.121668:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 298 00000400:00000200:3.0:1773941529.121672:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941529.121676:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941529.121680:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941529.121683:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941529.121685:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941529.121687:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941529.121690:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941529.121695:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941529.121697:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941529.121699:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941529.121722:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941529.121724:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941529.121727:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941529.121729:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941529.121731:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941529.121732:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941529.121735:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941529.121738:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941529.121740:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941529.121742:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941529.121745:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941529.121747:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941529.121749:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941529.121751:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941529.121754:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90274a80. 00000020:00000040:3.0:1773941529.121756:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941529.121759:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941529.121761:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941529.121763:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941529.121764:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941529.121766:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941529.121770:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90263000. 00000100:00000001:3.0:1773941529.121773:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941529.121779:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90274a80 x1860112241166592/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941529.121786:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941529.121788:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941529.121792:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e6e83776] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90274a80 x1860112241166592/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941529.121821:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941529.121825:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941529.121827:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941529.121830:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941529.121833:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941529.121835:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941529.121837:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941529.121840:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941529.121843:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941529.121851:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241166464/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941545 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941529.121860:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941529.121864:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941529.121866:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941529.121869:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941529.121875:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241166464/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941545 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941529.121877:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941529.121881:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941529.121883:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:3.0:1773941529.121886:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241166464:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000040:0.0:1773941529.121890:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90274a80 x1860112241166592/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941529.121891:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941529.121894:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941529.121897:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941529.121899:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941529.121900:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941529.121902:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941529.121904:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241166464/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941545 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1773941529.121905:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941529.121907:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941529.121910:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941529.121913:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941529.121914:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90274a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241166592:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000010:3.0:1773941529.121917:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:0.0:1773941529.121919:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1773941529.121920:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941529.121922:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:0.0:1773941529.121922:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941529.121924:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941529.121926:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:0.0:1773941529.121926:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941529.121932:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0000. 00010000:00000001:3.0:1773941529.121936:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941529.121938:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941529.121941:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 02000000:00000001:0.0:1773941529.121941:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941529.121944:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000200:0.0:1773941529.121946:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941529.121947:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000010:0.0:1773941529.121968:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00010000:3.0:1773941529.121970:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e3c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941529.121973:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941529.121978:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241166592, portal 25 00000100:00000001:0.0:1773941529.121981:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:3.0:1773941529.121982:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941529.121983:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109619840 : -128249599931776 : ffff8b5b90274a80) 00010000:00010000:3.0:1773941529.121988:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e3c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941529.121990:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90274a80 x1860112241166592/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941529.121999:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:0.0:1773941529.122002:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:3.0:1773941529.122003:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e3c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941529.122006:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941529.122009:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6ca18. 00010000:00000001:3.0:1773941529.122013:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000200:0.0:1773941529.122014:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241166592, offset 0 00010000:00000001:3.0:1773941529.122015:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:3.0:1773941529.122019:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e3c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941529.122020:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941529.122027:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:0.0:1773941529.122027:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941529.122030:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941529.122033:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:3.0:1773941529.122034:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:0.0:1773941529.122037:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:3.0:1773941529.122039:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e3c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941529.122041:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941529.122045:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:3.0:1773941529.122049:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:0.0:1773941529.122049:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000001:3.0:1773941529.122053:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941529.122053:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[174, 175] 10000000:00010000:3.0:1773941529.122057:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e3c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941529.122059:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941529.122064:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [281:281:256:4294967295] 192.168.204.143@tcp LPNI seq info [176:176:8:4294967295] 00000400:00000200:0.0:1773941529.122071:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941529.122078:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:3.0:1773941529.122080:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941529.122082:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1773941529.122082:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000020:00000040:3.0:1773941529.122085:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000020:00000001:3.0:1773941529.122088:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000800:00000200:0.0:1773941529.122089:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941529.122091:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000800:00000010:0.0:1773941529.122094:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b91e1e300. 00010000:00000001:3.0:1773941529.122095:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:3.0:1773941529.122099:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e3c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941529.122100:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00000001:3.0:1773941529.122108:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:0.0:1773941529.122109:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 10000000:00010000:3.0:1773941529.122112:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e3c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941529.122113:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b91e1e300 type 1, nob 616 niov 1 nkiov 1 10000000:01000000:3.0:1773941529.122121:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:3.0:1773941529.122124:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941529.122127:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941529.122130:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:0.0:1773941529.122131:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941529.122133:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:3.0:1773941529.122137:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941529.122139:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941529.122141:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:0.0:1773941529.122142:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941529.122144:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941529.122145:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294969485 00000020:00000040:3.0:1773941529.122148:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1e3c from hash 00010000:00000001:3.0:1773941529.122152:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:0.0:1773941529.122152:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294969483 00010000:00000001:3.0:1773941529.122154:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941529.122156:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941529.122158:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:0.0:1773941529.122158:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294969485 00010000:00000001:3.0:1773941529.122161:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941529.122163:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:0.0:1773941529.122163:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294969483 00010000:00000001:3.0:1773941529.122167:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941529.122168:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1773941529.122169:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941529.122172:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941529.122173:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941529.122175:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941529.122180:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941529.122181:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941529.122186:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e3c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941529.122189:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941529.122191:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:3.0:1773941529.122195:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941529.122195:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:3.0:1773941529.122197:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941529.122198:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:3.0:1773941529.122200:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:0.0:1773941529.122200:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941529.122202:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:0.0:1773941529.122203:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941529.122205:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941529.122206:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941529.122209:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941529.122213:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e3c lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:3.0:1773941529.122248:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962c00 count: 0 00000800:00000200:2.0:1773941529.122275:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:3.0:1773941529.122278:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:2.0:1773941529.122281:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b91e1e300. 00010000:00000001:3.0:1773941529.122284:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941529.122287:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941529.122289:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941529.122297:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241166464/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941545 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941529.122300:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941529.122308:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6ca18 00000400:00000010:2.0:1773941529.122313:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6ca18. 00000100:00000001:2.0:1773941529.122318:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941529.122323:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90274a80 x1860112241166592/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941529.122333:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941529.122337:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90274a80 x1860112241166592/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941539 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941529.122345:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941529.122347:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1773941529.122358:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941529.122362:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941529.122365:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941529.122368:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941529.122370:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941529.122372:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941529.122374:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941529.122376:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5ba0ab8000. 02000000:00000001:3.0:1773941529.122379:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941529.122380:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941529.122381:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:3.0:1773941529.122384:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941529.122385:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941529.122387:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941529.122389:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941529.122391:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941529.122403:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068600. 00000020:00000010:3.0:1773941529.122409:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab8000. 00000020:00000010:3.0:1773941529.122414:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab8a00. 00000040:00000010:3.0:1773941529.122419:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab8e00. 00000020:00000010:3.0:1773941529.122424:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941529.122429:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab8400. 80000000:00000010:3.0:1773941529.122435:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941529.122441:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab8800. 20000000:00000010:3.0:1773941529.122447:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5ba0ab9c00. 10000000:00000040:3.0:1773941529.122452:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:3.0:1773941529.122455:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941529.122461:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab8a00. 00000040:00000010:3.0:1773941529.122466:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab8e00. 00000020:00000010:3.0:1773941529.122469:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941529.122475:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab8400. 80000000:00000010:3.0:1773941529.122478:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941529.122483:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab8800. 20000000:00000010:3.0:1773941529.122486:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5ba0ab9c00. 00000020:00000010:3.0:1773941529.122490:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab8000. 10000000:00000010:3.0:1773941529.122493:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068600. 10000000:01000000:3.0:1773941529.122495:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941529.122498:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941529.122500:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941529.122502:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941529.122503:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941529.122504:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941529.122506:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941529.124162:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941529.124172:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941529.124176:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941529.124195:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941529.124201:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941529.124207:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941529.124231:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941529.124237:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941529.124241:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941529.124246:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608900 00000400:00000200:2.0:1773941529.124251:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941529.124256:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1345 [1] + 0 00000800:00000001:2.0:1773941529.124263:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941529.124273:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941529.124276:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941529.124281:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941529.124285:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941529.124299:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90274a80 x1860112241166592/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941539 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941529.124321:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90274a80 x1860112241166592/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941539 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941529.124342:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941529.124349:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941529.124354:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941529.124376:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941529.124380:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941529.124384:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941529.124385:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941529.124387:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941529.124402:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941529.124404:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941529.124406:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941529.124412:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90274a80 x1860112241166592/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941539 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941529.124422:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90274a80 x1860112241166592/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941539 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941529.124429:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941529.124430:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941529.124432:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941529.124433:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941529.124435:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941529.124436:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941529.124438:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941529.124441:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=108 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941529.124452:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941529.124454:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941529.124455:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941529.124457:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941529.124459:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941529.124461:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941529.124465:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90274a80 x1860112241166592/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941539 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941529.124472:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941529.124474:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941529.124475:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941529.124477:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941529.124481:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941529.124483:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941529.124484:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941529.124486:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941529.124488:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941529.124490:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941529.124492:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941529.124495:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941529 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941529.124497:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941529.124503:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941529.124505:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941529.124506:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941529.124508:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941529.124509:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941529.124512:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941529.124514:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941529.124516:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941529.124519:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941529.124521:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941529.124523:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941529.124786:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941529.124791:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941529.124800:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90274a80 x1860112241166592/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941539 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941529.124804:0:12249:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:1.0:1773941529.124811:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90274a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241166592:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941529.124819:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941529.124823:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1773941529.124823:0:12249:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:0.0:1773941529.124829:0:12249:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:1.0:1773941529.124831:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90274a80 x1860112241166592/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941539 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941529.124834:0:12249:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941529.124837:0:12249:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941529.124842:0:12249:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:1.0:1773941529.124843:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941529.124846:0:12249:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:1.0:1773941529.124847:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941529.124853:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0000. 00000001:00000001:0.0:1773941529.124853:0:12249:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941529.124858:0:12249:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:1.0:1773941529.124859:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941529.124862:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941529.124864:0:12249:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:1.0:1773941529.124865:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941529.124870:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:0.0:1773941529.124871:0:12249:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941529.124872:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90263000. 10000000:00000001:0.0:1773941529.124878:0:12249:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941529.124879:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941529.124881:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941529.124886:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90274a80. 00000100:00000001:0.0:1773941529.124886:0:12249:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941529.124889:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:0.0:1773941529.124894:0:12249:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941529.124900:0:12249:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941529.124901:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941529.124905:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941529.124906:0:12249:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941529.124910:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941529.124912:0:12249:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941529.124914:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941529.124918:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1773941529.124919:0:12249:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941529.124921:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941529.124927:0:12249:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941529.124929:0:12249:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941529.124931:0:12249:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941529.124954:0:12249:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941529.124956:0:12249:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941529.124959:0:12249:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941529.124964:0:12249:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941529.124968:0:12249:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941529.124970:0:12249:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941529.124971:0:12249:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941529.124973:0:12249:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941529.124975:0:12249:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941529.127765:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962c00. 00010000:00000010:3.1:1773941529.127776:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941529.440073:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941529.440081:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941529.440087:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941529.503933:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (304) 00000020:00000001:3.0:1773941530.463875:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941530.463881:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941530.463885:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941530.591797:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941530.591819:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941530.591824:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941530.591827:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941530.591829:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941530.591831:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941530.591834:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941530.591836:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941530.591839:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941530.591842:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941530.591844:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941530.591853:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941530.591858:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941530.591864:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941530.591868:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941530.591874:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941530.591884:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5ba0ab9c00. 00000100:00000001:3.0:1773941530.591892:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941530.591896:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941530.591899:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941530.591908:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962c00. 00010000:00000001:3.0:1773941530.591915:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941530.591918:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941530.591922:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941530.591924:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1e43 to hash 00000020:00000001:3.0:1773941530.591926:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941530.591927:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941530.591930:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941530.591933:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941530.591936:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941530.591940:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e43 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941530.591948:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e43 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941530.591968:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e43 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941530.591973:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941530.591975:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941530.591980:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941530.591982:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941530.591984:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941530.591986:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941530.591990:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941530.591994:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941530.592012:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941530.592014:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941530.592024:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241166720/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941530.592033:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941530.592035:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941530.592038:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941530.592041:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941530.592043:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941530.592047:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241166720:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941530.592050:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941530.592056:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941530.592058:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941530.592060:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941530.592065:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941530.592067:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941530.592074:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941530.592077:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941530.592084:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941530.592090:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241166720, portal 25 00000100:00000001:3.0:1773941530.592092:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941530.592097:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941530.592102:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241166720/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941546 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941530.592110:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941530.592124:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941530.592126:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a4c8. 00000100:00000200:3.0:1773941530.592129:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241166720, offset 0 00000400:00000200:3.0:1773941530.592135:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941530.592142:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941530.592149:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941530.592154:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941530.592158:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941530.592162:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941530.592165:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941530.592168:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[176, 175] 00000400:00000200:3.0:1773941530.592172:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941530.592176:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [282:282:256:4294967295] 192.168.204.153@tcp LPNI seq info [177:177:8:4294967295] 00000400:00000200:3.0:1773941530.592180:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941530.592214:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941530.592236:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941530.592250:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69f00. 00000800:00000200:3.0:1773941530.592256:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941530.592262:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941530.592267:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69f00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941530.592284:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941530.592288:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941530.592290:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294970955 00000100:00000001:3.0:1773941530.592296:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294969485 00000100:00000001:3.0:1773941530.592301:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294970955 00000100:00000001:3.0:1773941530.592303:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294969485 00000100:00000001:3.0:1773941530.592311:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941530.592313:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941530.592315:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941530.592316:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941530.592319:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941530.592321:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941530.592324:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941530.592428:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941530.592450:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69f00. 00000400:00000200:2.0:1773941530.592455:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941530.592462:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941530.592467:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a4c8 00000400:00000010:2.0:1773941530.592469:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a4c8. 00000100:00000001:2.0:1773941530.592474:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941530.592479:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241166720/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941546 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941530.592490:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941530.592493:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241166720/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941546 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941530.592500:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941530.592502:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941530.593574:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941530.593586:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941530.593589:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941530.593590:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941530.593595:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941530.593599:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941530.593605:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941530.593608:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941530.593612:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941530.593616:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608980 00000400:00000200:2.0:1773941530.593619:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941530.593632:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x134d [1] + 224 00000800:00000001:2.0:1773941530.593639:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941530.593646:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941530.593649:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941530.593652:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941530.593656:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941530.593660:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241166720/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941546 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941530.593669:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241166720/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941546 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941530.593685:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941530.593692:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941530.593694:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941530.593726:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941530.593733:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941530.593736:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941530.593738:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941530.593742:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941530.593744:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941530.593747:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941530.593752:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241166720/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941546 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941530.593761:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241166720/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941546 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941530.593767:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941530.593768:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941530.593771:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941530.593772:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941530.593774:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941530.593776:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941530.593778:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941530.593783:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=109 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941530.593786:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941530.593788:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941530.593791:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941530.593793:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941530.593796:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941530.593799:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941530.593802:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941530.593805:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941530.593815:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941530.593817:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941530.593819:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941530.593821:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941530.593824:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241166720/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941546 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941530.593832:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941530.593834:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941530.593836:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941530.593839:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941530.593842:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941530.593844:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941530.593846:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941530.593848:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941530.593849:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941530.593851:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941530.593852:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941530.593855:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941530.593857:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941530.593861:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941530.593865:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 299 00000400:00000200:3.0:1773941530.593871:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941530.593876:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941530.593880:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941530.593884:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941530.593887:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941530.593889:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941530.593893:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941530.593895:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941530.593898:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941530.593900:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941530.593903:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941530.593905:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941530.593916:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941530.593919:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941530.593921:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941530.593923:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941530.593926:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941530.593930:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941530.593933:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941530.593936:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941530.593940:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941530.593943:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941530.593945:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941530.593947:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941530.593953:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90274a80. 00000020:00000040:3.0:1773941530.593956:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941530.593959:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941530.593961:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941530.593964:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941530.593966:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941530.593968:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941530.593973:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90262000. 00000100:00000001:3.0:1773941530.593977:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941530.593983:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90274a80 x1860112241166848/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941530.593991:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941530.593995:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941530.594000:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e6e83776] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90274a80 x1860112241166848/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941530.594034:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941530.594037:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941530.594039:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941530.594042:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941530.594044:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941530.594053:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941530.594055:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941530.594057:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941530.594060:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941530.594066:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241166720/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941546 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941530.594073:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941530.594077:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941530.594079:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941530.594081:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941530.594086:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241166720/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941546 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941530.594095:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241166720:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941530.594101:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941530.594104:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941530.594107:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941530.594109:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941530.594113:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241166720/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941546 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941530.594119:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941530.594121:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941530.594126:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000010:3.0:1773941530.594127:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:1.0:1773941530.594132:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1773941530.594133:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941530.594138:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:3.0:1773941530.594141:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:3.0:1773941530.594147:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000040:1.0:1773941530.594147:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90274a80 x1860112241166848/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1773941530.594149:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941530.594153:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 00000020:00000001:3.0:1773941530.594158:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941530.594164:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00010000:3.0:1773941530.594172:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e43 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941530.594172:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941530.594175:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941530.594180:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941530.594185:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1773941530.594206:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:1.0:1773941530.594206:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941530.594217:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e43 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1773941530.594217:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90274a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241166848:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941530.594230:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:3.0:1773941530.594233:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000001:1.0:1773941530.594235:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00010000:3.0:1773941530.594238:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e43 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:1.0:1773941530.594239:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941530.594244:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941530.594250:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0735400. 00010000:00000001:3.0:1773941530.594252:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000001:1.0:1773941530.594254:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941530.594256:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:3.0:1773941530.594260:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e43 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941530.594269:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941530.594275:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00010000:00000001:3.0:1773941530.594277:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000010:1.0:1773941530.594280:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00010000:00000001:3.0:1773941530.594281:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941530.594287:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000200:1.0:1773941530.594288:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241166848, portal 25 00010000:00010000:3.0:1773941530.594292:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e43 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941530.594292:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941530.594296:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109619840 : -128249599931776 : ffff8b5b90274a80) 00000100:00000040:1.0:1773941530.594306:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90274a80 x1860112241166848/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1773941530.594307:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:3.0:1773941530.594313:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941530.594316:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e43 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941530.594322:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:3.0:1773941530.594328:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000040:1.0:1773941530.594328:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941530.594331:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941530.594332:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941530.594333:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000020:00000001:3.0:1773941530.594335:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941530.594340:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000400:00000010:1.0:1773941530.594340:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1908. 00000100:00000200:1.0:1773941530.594345:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241166848, offset 0 00000400:00000200:1.0:1773941530.594351:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941530.594353:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:3.0:1773941530.594357:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e43 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941530.594358:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941530.594366:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 10000000:00000001:3.0:1773941530.594367:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941530.594371:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e43 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941530.594373:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941530.594380:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 10000000:01000000:3.0:1773941530.594383:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:1.0:1773941530.594386:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000040:3.0:1773941530.594388:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:1.0:1773941530.594391:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000001:3.0:1773941530.594392:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941530.594398:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:1.0:1773941530.594398:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[176, 177] 00010000:00000001:3.0:1773941530.594400:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:3.0:1773941530.594405:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:1.0:1773941530.594406:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:3.0:1773941530.594408:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941530.594412:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:1.0:1773941530.594413:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [283:283:256:4294967295] 192.168.204.143@tcp LPNI seq info [178:178:8:4294967295] 00010000:00000001:3.0:1773941530.594415:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941530.594420:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1e43 from hash 00000400:00000200:1.0:1773941530.594424:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941530.594425:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941530.594429:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941530.594450:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941530.594453:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:3.0:1773941530.594457:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:1.0:1773941530.594461:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:3.0:1773941530.594462:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:3.0:1773941530.594468:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941530.594471:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941530.594474:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941530.594474:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 10000000:00000001:3.0:1773941530.594477:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1773941530.594481:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024bb00. 00010000:00010000:3.0:1773941530.594482:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e43 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941530.594492:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:1.0:1773941530.594493:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:3.0:1773941530.594495:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941530.594499:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941530.594501:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000800:00000200:1.0:1773941530.594505:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:3.0:1773941530.594507:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941530.594512:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:1.0:1773941530.594513:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024bb00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:3.0:1773941530.594516:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941530.594520:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e43 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:3.0:1773941530.594526:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962c00 count: 0 00000100:00000001:1.0:1773941530.594530:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941530.594537:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:1.0:1773941530.594537:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941530.594539:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294970957 00010000:00000001:3.0:1773941530.594541:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941530.594546:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941530.594547:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294970955 00000100:00000040:3.0:1773941530.594553:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241166720/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941546 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941530.594554:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294970957 00000100:00000001:3.0:1773941530.594566:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941530.594571:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1773941530.594572:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294970955 02000000:00000010:3.0:1773941530.594575:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 00000100:00000001:1.0:1773941530.594578:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941530.594579:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941530.594582:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941530.594583:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941530.594588:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941530.594589:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941530.594595:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941530.594597:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1773941530.594600:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5ba0ab9c00. 02000000:00000001:3.0:1773941530.594604:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941530.594605:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1773941530.594607:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941530.594609:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:3.0:1773941530.594612:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941530.594614:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941530.594616:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 10000000:00000001:3.0:1773941530.594617:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941530.594622:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:01000000:3.0:1773941530.594623:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:1.0:1773941530.594627:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000001:3.0:1773941530.594629:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:1.0:1773941530.594631:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941530.594634:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000010:3.0:1773941530.594635:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068600. 00000020:00000010:3.0:1773941530.594642:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab9c00. 00000020:00000010:3.0:1773941530.594648:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab8800. 00000040:00000010:3.0:1773941530.594652:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab8400. 00000020:00000010:3.0:1773941530.594657:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000800:00000200:2.0:1773941530.594660:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000020:00000010:3.0:1773941530.594662:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab8e00. 00000800:00000010:2.0:1773941530.594666:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024bb00. 80000000:00000010:3.0:1773941530.594668:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00000400:00000200:2.0:1773941530.594674:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00040000:00000010:3.0:1773941530.594683:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab8a00. 00000400:00000200:2.0:1773941530.594691:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 20000000:00000010:3.0:1773941530.594692:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5ba0ab9200. 10000000:00000040:3.0:1773941530.594700:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000400:00000200:2.0:1773941530.594700:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1908 10000000:00000001:3.0:1773941530.594704:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941530.594709:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab8800. 00000040:00000010:3.0:1773941530.594730:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab8400. 00000020:00000010:3.0:1773941530.594733:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000400:00000010:2.0:1773941530.594749:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1908. 00000100:00000001:2.0:1773941530.594754:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941530.594759:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90274a80 x1860112241166848/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941530.594767:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941530.594769:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90274a80 x1860112241166848/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941540 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941530.594775:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941530.594776:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:3.0:1773941530.594822:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab8e00. 80000000:00000010:3.0:1773941530.594829:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941530.594834:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab8a00. 20000000:00000010:3.0:1773941530.594838:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5ba0ab9200. 00000020:00000010:3.0:1773941530.594842:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab9c00. 10000000:00000010:3.0:1773941530.594851:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068600. 10000000:01000000:3.0:1773941530.594855:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941530.594859:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941530.594862:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941530.594864:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941530.594865:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941530.594867:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941530.594870:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941530.596483:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941530.596492:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941530.596495:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941530.596498:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941530.596503:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941530.596517:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941530.596523:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941530.596527:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941530.596532:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941530.596536:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608a00 00000400:00000200:2.0:1773941530.596542:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941530.596547:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x135d [1] + 0 00000800:00000001:2.0:1773941530.596553:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941530.596561:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941530.596564:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941530.596568:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941530.596572:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941530.596578:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90274a80 x1860112241166848/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941540 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941530.596592:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90274a80 x1860112241166848/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941540 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941530.596611:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941530.596617:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941530.596620:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941530.596641:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941530.596645:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941530.596648:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941530.596649:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941530.596651:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941530.596654:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941530.596656:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941530.596658:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941530.596663:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90274a80 x1860112241166848/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941540 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941530.596672:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90274a80 x1860112241166848/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941540 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941530.596678:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941530.596679:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941530.596680:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941530.596682:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941530.596683:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941530.596684:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941530.596686:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941530.596689:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=109 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941530.596692:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941530.596710:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941530.596712:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941530.596713:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941530.596715:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941530.596732:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941530.596736:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90274a80 x1860112241166848/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941540 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941530.596741:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941530.596743:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941530.596751:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941530.596753:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1773941530.596757:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:1.0:1773941530.596759:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941530.596761:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941530.596763:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941530.596765:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941530.596766:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941530.596769:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:1.0:1773941530.596772:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941530 v=0 (0 0 0 0) 00000100:00001000:1.0:1773941530.596774:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:1.0:1773941530.596779:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:1.0:1773941530.596780:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941530.596782:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:1.0:1773941530.596783:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941530.596785:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941530.596788:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:1.0:1773941530.596790:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:1.0:1773941530.596791:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:1.0:1773941530.596794:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:1.0:1773941530.596796:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:1.0:1773941530.596798:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941530.597079:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941530.597083:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941530.597090:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90274a80 x1860112241166848/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941540 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941530.597099:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90274a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241166848:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941530.597102:0:12251:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:1.0:1773941530.597111:0:12251:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:0.0:1773941530.597114:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941530.597115:0:12251:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941530.597118:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941530.597119:0:12251:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941530.597122:0:12251:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000040:0.0:1773941530.597125:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90274a80 x1860112241166848/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941540 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:1.0:1773941530.597127:0:12251:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941530.597133:0:12251:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:0.0:1773941530.597135:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941530.597138:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941530.597140:0:12251:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941530.597141:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0735400. 00000001:00000001:1.0:1773941530.597146:0:12251:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:0.0:1773941530.597147:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941530.597151:0:12251:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:0.0:1773941530.597151:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941530.597154:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:1.0:1773941530.597157:0:12251:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941530.597157:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941530.597160:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90262000. 10000000:00000001:1.0:1773941530.597163:0:12251:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941530.597164:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941530.597165:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941530.597166:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90274a80. 00000100:00000001:0.0:1773941530.597169:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941530.597169:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941530.597173:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941530.597175:0:12251:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941530.597178:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1773941530.597180:0:12251:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941530.597181:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941530.597198:0:12251:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941530.597203:0:12251:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941530.597205:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941530.597206:0:12251:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941530.597207:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941530.597210:0:12251:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941530.597214:0:12251:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941530.597215:0:12251:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941530.597216:0:12251:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941530.597229:0:12251:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941530.597231:0:12251:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941530.597232:0:12251:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941530.597235:0:12251:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941530.597237:0:12251:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941530.597239:0:12251:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941530.597240:0:12251:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941530.597242:0:12251:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941530.597244:0:12251:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941530.599742:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962c00. 00010000:00000010:3.1:1773941530.599761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941531.487866:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941531.487871:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941531.487875:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941531.999855:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941531.999863:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941531.999868:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941531.999871:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941531.999872:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941531.999873:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941531.999876:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941531.999878:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941531.999881:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:3.0:1773941531.999883:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941531.999885:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941531.999896:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941531.999917:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941531.999922:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941531.999925:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941531.999931:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941531.999938:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5ba0ab8400. 00000100:00000001:3.0:1773941531.999945:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941531.999949:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941531.999952:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941531.999960:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b83962c00. 00010000:00000001:3.0:1773941531.999965:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941531.999968:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941531.999972:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941531.999975:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b81485e00 with handle 0xa33a5be14eae1e4a to hash 00000020:00000001:3.0:1773941531.999977:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941531.999979:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941531.999982:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941531.999985:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941531.999988:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941531.999992:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e4a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941532.000002:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e4a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941532.000013:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e4a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941532.000018:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941532.000020:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941532.000025:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941532.000027:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387399872 : -128249322151744 : ffff8b5ba0b5e0c0) 00000100:00000001:3.0:1773941532.000030:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941532.000031:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000001:3.0:1773941532.000036:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941532.000047:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941532.000049:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941532.000051:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941532.000062:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241166976/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941532.000071:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941532.000073:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941532.000076:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941532.000079:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941532.000081:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941532.000084:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241166976:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941532.000088:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941532.000092:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941532.000094:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941532.000097:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941532.000101:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b90260c00. 02000000:00000001:3.0:1773941532.000103:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941532.000110:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:3.0:1773941532.000114:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b845ab6e0. 00000400:00000010:3.0:1773941532.000120:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a088. 00000100:00000200:3.0:1773941532.000126:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241166976, portal 25 00000100:00000001:3.0:1773941532.000128:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941532.000129:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00000100:00000040:3.0:1773941532.000134:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241166976/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941547 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941532.000141:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941532.000144:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:3.0:1773941532.000146:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5ba0b6a220. 00000100:00000200:3.0:1773941532.000149:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241166976, offset 0 00000400:00000200:3.0:1773941532.000154:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:3.0:1773941532.000197:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941532.000205:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:3.0:1773941532.000210:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941532.000220:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:3.0:1773941532.000224:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941532.000227:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941532.000229:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[178, 177] 00000400:00000200:3.0:1773941532.000233:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:3.0:1773941532.000237:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [284:284:256:4294967295] 192.168.204.153@tcp LPNI seq info [179:179:8:4294967295] 00000400:00000200:3.0:1773941532.000241:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:3.0:1773941532.000252:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:3.0:1773941532.000258:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:3.0:1773941532.000263:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98a69500. 00000800:00000200:3.0:1773941532.000270:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:3.0:1773941532.000277:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:3.0:1773941532.000282:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98a69500 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941532.000301:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941532.000305:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:3.0:1773941532.000308:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294972363 00000100:00000001:3.0:1773941532.000315:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294970957 00000100:00000001:3.0:1773941532.000319:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294972363 00000100:00000001:3.0:1773941532.000323:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294970957 00000100:00000001:3.0:1773941532.000325:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941532.000328:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941532.000330:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941532.000331:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941532.000334:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 11 seconds 00000100:00000001:3.0:1773941532.000337:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941532.000339:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941532.000513:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941532.000521:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98a69500. 00000400:00000200:2.0:1773941532.000529:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941532.000538:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941532.000545:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a220 00000400:00000010:2.0:1773941532.000549:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a220. 00000100:00000001:2.0:1773941532.000555:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941532.000563:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241166976/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941547 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941532.000578:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941532.000582:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90275180 x1860112241166976/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941547 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941532.000611:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941532.000614:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941532.001921:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941532.001933:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941532.001945:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941532.001947:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941532.001953:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941532.001957:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941532.001963:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941532.001967:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941532.001971:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941532.001975:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608a80 00000400:00000200:2.0:1773941532.001978:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941532.001983:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x1365 [1] + 224 00000800:00000001:2.0:1773941532.001990:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941532.001998:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941532.002001:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941532.002004:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941532.002009:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941532.002014:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241166976/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941547 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941532.002024:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90275180 x1860112241166976/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941547 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941532.002041:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941532.002048:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941532.002051:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941532.002076:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941532.002083:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941532.002085:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941532.002087:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab6e0. 00000400:00000200:3.0:1773941532.002092:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b6a088 00000400:00000010:3.0:1773941532.002101:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5ba0b6a088. 00000100:00000001:3.0:1773941532.002104:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941532.002110:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241166976/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941547 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941532.002120:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241166976/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941547 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941532.002128:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941532.002129:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941532.002132:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941532.002134:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941532.002136:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941532.002138:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941532.002140:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941532.002146:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=111 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941532.002149:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941532.002152:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941532.002156:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941532.002158:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941532.002176:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941532.002179:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941532.002183:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941532.002187:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941532.002190:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941532.002192:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941532.002194:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941532.002195:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941532.002199:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90275180 x1860112241166976/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941547 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941532.002213:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941532.002214:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941532.002216:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941532.002224:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941532.002226:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941532.002228:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941532.002230:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941532.002231:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941532.002232:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941532.002233:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941532.002235:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941532.002237:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941532.002239:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941532.002241:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941532.002244:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 301 00000400:00000200:3.0:1773941532.002248:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941532.002252:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941532.002256:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941532.002259:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941532.002261:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941532.002263:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941532.002266:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941532.002268:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941532.002269:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941532.002272:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941532.002273:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941532.002275:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941532.002278:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941532.002279:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941532.002281:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941532.002282:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941532.002285:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941532.002288:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941532.002290:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941532.002292:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941532.002295:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941532.002301:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941532.002303:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941532.002305:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941532.002308:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90277b80. 00000020:00000040:3.0:1773941532.002310:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941532.002313:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941532.002315:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941532.002317:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941532.002318:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941532.002321:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941532.002324:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90262000. 00000100:00000001:3.0:1773941532.002328:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941532.002333:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90277b80 x1860112241167104/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941532.002340:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941532.002343:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941532.002347:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000010dacf3] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90277b80 x1860112241167104/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941532.002369:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941532.002372:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941532.002373:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941532.002392:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941532.002394:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941532.002395:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941532.002397:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b5e0c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941532.002399:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941532.002401:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941532.002405:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241166976/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941547 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941532.002411:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941532.002413:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941532.002415:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941532.002417:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941532.002425:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241166976/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941547 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941532.002438:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241166976:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941532.002442:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941532.002444:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941532.002446:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941532.002447:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941532.002450:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241166976/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941547 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941532.002456:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941532.002458:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b5e0c0. 00000100:00000001:3.0:1773941532.002461:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941532.002463:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:3.0:1773941532.002466:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:3.0:1773941532.002468:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941532.002470:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941532.002471:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=4 00000020:00000001:3.0:1773941532.002474:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941532.002476:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00010000:3.0:1773941532.002480:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e4a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941532.002481:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941532.002487:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941532.002492:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00010000:00000001:3.0:1773941532.002493:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:3.0:1773941532.002500:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e4a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941532.002504:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90277b80 x1860112241167104/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941532.002512:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:3.0:1773941532.002517:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e4a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941532.002522:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941532.002526:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941532.002531:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941532.002534:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941532.002539:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941532.002547:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90277b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241167104:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:3.0:1773941532.002551:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941532.002553:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:3.0:1773941532.002555:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:3.0:1773941532.002558:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e4a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941532.002563:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941532.002568:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941532.002569:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:3.0:1773941532.002571:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 02000000:00000001:0.0:1773941532.002572:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941532.002577:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e4a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941532.002578:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941532.002584:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0c00. 00010000:00010000:3.0:1773941532.002587:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b81485e00 already unused, calling callback (ffffffffc1482c20) 02000000:00000001:0.0:1773941532.002588:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941532.002593:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941532.002597:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00010000:3.0:1773941532.002598:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e4a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941532.002601:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00000001:3.0:1773941532.002610:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000010:0.0:1773941532.002610:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00010000:00000001:3.0:1773941532.002612:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941532.002616:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:0.0:1773941532.002617:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241167104, portal 25 00000020:00000040:3.0:1773941532.002618:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b81485e00 refcount=5 00000100:00000001:0.0:1773941532.002623:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1773941532.002624:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00000100:00000001:0.0:1773941532.002626:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109632384 : -128249599919232 : ffff8b5b90277b80) 00010000:00000001:3.0:1773941532.002629:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823860129280 : -128249849422336 : ffff8b5b81485e00) 00010000:00000001:3.0:1773941532.002636:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000040:0.0:1773941532.002636:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90277b80 x1860112241167104/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941542 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1773941532.002641:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e4a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941532.002662:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 10000000:00000001:3.0:1773941532.002664:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000040:0.0:1773941532.002669:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 10000000:00010000:3.0:1773941532.002670:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e4a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941532.002672:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6cc38. 00000100:00000200:0.0:1773941532.002679:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241167104, offset 0 10000000:01000000:3.0:1773941532.002680:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:0.0:1773941532.002685:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941532.002690:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000040:3.0:1773941532.002696:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:0.0:1773941532.002700:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 10000000:00000001:3.0:1773941532.002731:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941532.002735:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941532.002739:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:0.0:1773941532.002740:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:3.0:1773941532.002744:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:0.0:1773941532.002746:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:3.0:1773941532.002748:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:0.0:1773941532.002752:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:3.0:1773941532.002754:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941532.002758:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941532.002760:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1773941532.002765:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b81485e00 with handle 0xa33a5be14eae1e4a from hash 00000400:00000200:0.0:1773941532.002770:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[178, 179] 00010000:00000001:3.0:1773941532.002771:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941532.002775:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941532.002779:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:3.0:1773941532.002780:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941532.002784:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:3.0:1773941532.002788:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:0.0:1773941532.002788:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [285:285:256:4294967295] 192.168.204.143@tcp LPNI seq info [180:180:8:4294967295] 00010000:00000001:3.0:1773941532.002792:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:3.0:1773941532.002797:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941532.002798:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941532.002800:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941532.002803:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941532.002805:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941532.002811:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e4a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941532.002824:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:3.0:1773941532.002825:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941532.002829:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941532.002832:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941532.002835:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000800:00000200:0.0:1773941532.002836:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941532.002839:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941532.002842:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:0.0:1773941532.002843:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b91e1e300. 00010000:00000001:3.0:1773941532.002844:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941532.002849:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b81485e00/0xa33a5be14eae1e4a lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941532.002856:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000040:3.0:1773941532.002860:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b83962c00 count: 0 00010000:00000001:3.0:1773941532.002882:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:0.0:1773941532.002885:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:3.0:1773941532.002886:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000200:0.0:1773941532.002891:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b91e1e300 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:3.0:1773941532.002892:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941532.002907:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241166976/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941547 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941532.002908:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941532.002918:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941532.002923:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294972366 00000100:00000001:3.0:1773941532.002925:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941532.002929:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294972363 02000000:00000001:3.0:1773941532.002930:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941532.002943:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b90260c00. 00000100:00000001:0.0:1773941532.002945:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294972366 02000000:00000001:3.0:1773941532.002948:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1773941532.002951:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294972363 00000020:00000001:3.0:1773941532.002952:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941532.002956:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941532.002958:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1773941532.002959:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941532.002963:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941532.002964:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5ba0ab8400. 00000100:00000001:0.0:1773941532.002968:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941532.002970:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1773941532.002972:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1773941532.002973:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941532.002978:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:3.0:1773941532.002982:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941532.002983:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941532.002984:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000001:3.0:1773941532.002985:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941532.002988:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 10000000:01000000:3.0:1773941532.002990:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:0.0:1773941532.002993:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000001:3.0:1773941532.002995:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:0.0:1773941532.002997:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000010:3.0:1773941532.003001:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068600. 00000100:00000001:0.0:1773941532.003001:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941532.003007:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941532.003010:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab8400. 00000020:00000010:3.0:1773941532.003017:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab8800. 00000040:00000010:3.0:1773941532.003021:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab8600. 00000020:00000010:3.0:1773941532.003027:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941532.003031:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab9600. 80000000:00000010:3.0:1773941532.003035:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941532.003039:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab9a00. 20000000:00000010:3.0:1773941532.003044:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5ba0ab8200. 00000800:00000200:2.0:1773941532.003047:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 10000000:00000040:3.0:1773941532.003048:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:3.0:1773941532.003053:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000800:00000010:2.0:1773941532.003053:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b91e1e300. 00000020:00000010:3.0:1773941532.003062:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab8800. 00000400:00000200:2.0:1773941532.003062:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000040:00000010:3.0:1773941532.003071:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab8600. 00000400:00000200:2.0:1773941532.003074:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000020:00000010:3.0:1773941532.003076:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000400:00000200:2.0:1773941532.003079:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6cc38 00000020:00000010:3.0:1773941532.003082:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab9600. 80000000:00000010:3.0:1773941532.003085:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00000400:00000010:2.0:1773941532.003092:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6cc38. 00040000:00000010:3.0:1773941532.003100:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab9a00. 00000100:00000001:2.0:1773941532.003100:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941532.003108:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90277b80 x1860112241167104/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941542 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941532.003116:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941532.003119:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90277b80 x1860112241167104/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941542 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941532.003126:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941532.003127:0:2347:0:(events.c:68:request_out_callback()) Process leaving 20000000:00000010:3.0:1773941532.003153:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5ba0ab8200. 00000020:00000010:3.0:1773941532.003156:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab8400. 10000000:00000010:3.0:1773941532.003159:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068600. 10000000:01000000:3.0:1773941532.003180:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941532.003183:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941532.003185:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941532.003187:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941532.003188:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941532.003189:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:3.0:1773941532.003191:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941532.005258:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941532.005267:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941532.005270:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941532.005272:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941532.005277:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941532.005283:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941532.005289:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941532.005293:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941532.005296:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941532.005299:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608b00 00000400:00000200:2.0:1773941532.005303:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941532.005306:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1375 [1] + 0 00000800:00000001:2.0:1773941532.005311:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941532.005317:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941532.005319:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941532.005322:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941532.005326:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941532.005331:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90277b80 x1860112241167104/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941542 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941532.005341:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90277b80 x1860112241167104/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941542 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941532.005361:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941532.005374:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941532.005377:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941532.005421:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941532.005426:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941532.005431:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941532.005433:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941532.005436:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941532.005442:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941532.005445:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941532.005449:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941532.005457:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90277b80 x1860112241167104/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941542 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941532.005472:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90277b80 x1860112241167104/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941542 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941532.005483:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941532.005485:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941532.005488:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941532.005501:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941532.005503:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941532.005506:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941532.005509:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941532.005515:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=111 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941532.005519:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941532.005523:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941532.005525:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941532.005529:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941532.005531:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941532.005535:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941532.005543:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90277b80 x1860112241167104/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941542 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941532.005552:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941532.005555:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941532.005558:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941532.005561:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941532.005566:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941532.005569:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941532.005571:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941532.005573:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941532.005576:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941532.005579:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941532.005583:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941532.005587:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941532 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941532.005592:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941532.005599:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941532.005602:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941532.005605:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941532.005607:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941532.005610:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941532.005614:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941532.005617:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941532.005620:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941532.005631:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941532.005636:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941532.005639:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941532.005962:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941532.005969:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941532.005978:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90277b80 x1860112241167104/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941542 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941532.005992:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90277b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241167104:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941532.005997:0:12252:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:1.0:1773941532.006000:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941532.006006:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1773941532.006013:0:12252:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000040:1.0:1773941532.006015:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90277b80 x1860112241167104/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941542 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941532.006021:0:12252:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941532.006027:0:12252:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941532.006031:0:12252:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:1.0:1773941532.006033:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941532.006037:0:12252:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:1.0:1773941532.006038:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941532.006044:0:12252:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:1.0:1773941532.006045:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0c00. 00000001:00000001:0.0:1773941532.006051:0:12252:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941532.006052:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941532.006058:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941532.006059:0:12252:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000040:1.0:1773941532.006062:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941532.006063:0:12252:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941532.006066:0:12252:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941532.006069:0:12252:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941532.006071:0:12252:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:1.0:1773941532.006078:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:0.0:1773941532.006078:0:12252:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:1.0:1773941532.006091:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90262000. 00000100:00000001:0.0:1773941532.006093:0:12252:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:1.0:1773941532.006098:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1773941532.006098:0:12252:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:1.0:1773941532.006104:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941532.006105:0:12252:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941532.006111:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90277b80. 00000100:00080000:0.0:1773941532.006112:0:12252:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941532.006122:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941532.006122:0:12252:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:1.0:1773941532.006127:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:0.0:1773941532.006128:0:12252:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941532.006134:0:12252:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:1.0:1773941532.006135:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941532.006143:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941532.006145:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941532.006148:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941532.006151:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941532.006179:0:12252:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941532.006184:0:12252:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941532.006188:0:12252:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941532.006194:0:12252:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941532.006199:0:12252:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941532.006203:0:12252:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941532.006205:0:12252:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941532.006208:0:12252:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941532.006211:0:12252:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941532.008781:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b83962c00. 00010000:00000010:3.1:1773941532.008790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b81485e00. 00000020:00000001:3.0:1773941532.511926:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941532.511933:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941532.511938:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941533.535845:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941533.535853:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941533.535857:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941533.855813:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941533.855820:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941533.855824:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941533.855826:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941533.855827:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941533.855828:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941533.855830:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941533.855832:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941533.855834:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941533.855836:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941533.855837:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941533.855846:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60e00. 00000020:00000040:1.0:1773941533.855850:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941533.855854:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941533.855857:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941533.855873:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941533.855879:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941533.855884:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941533.855888:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941533.855890:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941533.855897:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec600. 00010000:00000001:1.0:1773941533.855901:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941533.855905:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138ec00. 00000020:00000001:1.0:1773941533.855908:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941533.855911:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138ec00 with handle 0xa33a5be14eae1e51 to hash 00000020:00000001:1.0:1773941533.855912:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941533.855926:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941533.855928:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941533.855943:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941533.855946:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941533.855949:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e51 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941533.855957:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e51 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941533.855966:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e51 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941533.855971:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941533.855973:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941533.855977:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941533.855979:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941533.855981:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941533.855982:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00000100:00000001:1.0:1773941533.855986:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941533.855990:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941533.855992:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941533.855993:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941533.856002:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60e00 x1860112241167232/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941533.856008:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941533.856010:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941533.856013:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941533.856014:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941533.856016:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941533.856018:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241167232:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941533.856021:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941533.856028:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941533.856030:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941533.856032:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941533.856052:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734000. 02000000:00000001:1.0:1773941533.856054:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941533.856061:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941533.856065:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941533.856072:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941533.856079:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241167232, portal 25 00000100:00000001:1.0:1773941533.856081:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941533.856083:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00000100:00000040:1.0:1773941533.856088:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60e00 x1860112241167232/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941549 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941533.856096:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941533.856099:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941533.856101:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1b28. 00000100:00000200:1.0:1773941533.856104:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241167232, offset 0 00000400:00000200:1.0:1773941533.856111:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941533.856118:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941533.856179:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941533.856184:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941533.856187:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941533.856191:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941533.856193:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941533.856195:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[180, 179] 00000400:00000200:1.0:1773941533.856198:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941533.856201:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [286:286:256:4294967295] 192.168.204.153@tcp LPNI seq info [181:181:8:4294967295] 00000400:00000200:1.0:1773941533.856204:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941533.856212:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941533.856216:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941533.856220:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b600. 00000800:00000200:1.0:1773941533.856226:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941533.856236:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941533.856240:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941533.856258:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941533.856263:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941533.856266:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294974219 00000100:00000001:1.0:1773941533.856272:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294972366 00000100:00000001:1.0:1773941533.856276:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294974219 00000100:00000001:1.0:1773941533.856280:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294972366 00000100:00000001:1.0:1773941533.856284:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941533.856287:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941533.856290:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941533.856293:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941533.856296:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941533.856299:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941533.856302:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941533.856387:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941533.856392:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b600. 00000400:00000200:2.0:1773941533.856399:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941533.856422:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941533.856429:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1b28 00000400:00000010:2.0:1773941533.856433:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1b28. 00000100:00000001:2.0:1773941533.856438:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941533.856445:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60e00 x1860112241167232/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941549 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941533.856468:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941533.856473:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d60e00 x1860112241167232/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941549 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941533.856484:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941533.856487:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941533.857558:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941533.857569:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941533.857572:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941533.857575:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941533.857581:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941533.857586:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941533.857593:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941533.857597:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941533.857601:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941533.857604:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608b80 00000400:00000200:2.0:1773941533.857607:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941533.857611:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x137d [1] + 224 00000800:00000001:2.0:1773941533.857616:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941533.857623:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941533.857625:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941533.857627:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941533.857631:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941533.857640:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60e00 x1860112241167232/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941549 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941533.857648:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d60e00 x1860112241167232/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941549 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941533.857662:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941533.857668:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941533.857671:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941533.857717:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941533.857723:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941533.857724:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941533.857727:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941533.857730:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941533.857731:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941533.857733:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941533.857738:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60e00 x1860112241167232/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941549 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941533.857746:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60e00 x1860112241167232/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941549 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941533.857751:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941533.857752:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941533.857754:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941533.857756:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941533.857758:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941533.857760:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941533.857761:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941533.857772:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=112 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941533.857774:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941533.857776:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941533.857779:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941533.857781:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941533.857784:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941533.857786:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941533.857789:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941533.857792:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941533.857795:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941533.857796:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941533.857798:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941533.857799:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941533.857801:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d60e00 x1860112241167232/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941549 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941533.857806:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941533.857807:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941533.857808:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941533.857811:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941533.857814:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941533.857815:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941533.857817:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941533.857818:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941533.857819:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941533.857820:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941533.857822:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941533.857824:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941533.857826:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941533.857829:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941533.857832:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 302 00000400:00000200:1.0:1773941533.857837:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941533.857841:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941533.857845:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941533.857855:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941533.857858:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941533.857860:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941533.857863:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941533.857865:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941533.857868:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941533.857870:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941533.857872:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941533.857875:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941533.857879:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941533.857881:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941533.857883:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941533.857884:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941533.857886:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941533.857889:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941533.857891:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941533.857892:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941533.857894:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941533.857896:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941533.857898:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941533.857899:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941533.857902:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941533.857904:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941533.857906:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941533.857908:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941533.857909:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941533.857910:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941533.857912:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941533.857915:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734c00. 00000100:00000001:1.0:1773941533.857918:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941533.857923:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63800 x1860112241167360/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941533.857937:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941533.857940:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941533.857945:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc928f9c] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63800 x1860112241167360/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941533.857966:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941533.857969:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941533.857971:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941533.857974:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941533.857976:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941533.857978:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941533.857980:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941533.857982:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941533.857985:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941533.857992:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60e00 x1860112241167232/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941549 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941533.858000:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941533.858002:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941533.858004:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941533.858006:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941533.858009:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941533.858011:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941533.858033:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941533.858038:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60e00 x1860112241167232/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941549 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941533.858043:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241167360/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941533.858074:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941533.858076:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941533.858079:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241167232:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000001:0.0:1773941533.858080:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941533.858084:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941533.858091:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941533.858093:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941533.858096:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241167360:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941533.858100:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941533.858102:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941533.858106:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:0.0:1773941533.858107:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941533.858110:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1773941533.858111:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941533.858115:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941533.858120:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0c00. 00000100:00000040:1.0:1773941533.858121:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60e00 x1860112241167232/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941549 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1773941533.858123:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941533.858147:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941533.858152:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941533.858159:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000001:1.0:1773941533.858161:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1773941533.858165:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241167360, portal 25 00000100:00000001:0.0:1773941533.858167:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941533.858169:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:0.0:1773941533.858176:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241167360/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941533.858177:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941533.858185:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941533.858189:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1773941533.858191:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:0.0:1773941533.858194:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941533.858200:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000010:0.0:1773941533.858201:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c880. 00000100:00000200:0.0:1773941533.858206:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241167360, offset 0 00010000:00000001:1.0:1773941533.858208:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941533.858213:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1773941533.858214:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000020:00000040:1.0:1773941533.858220:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ec00 refcount=4 00000400:00000200:0.0:1773941533.858223:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000001:1.0:1773941533.858227:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00000400:00000200:0.0:1773941533.858231:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:1.0:1773941533.858233:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00000400:00000200:0.0:1773941533.858236:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941533.858241:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00010000:1.0:1773941533.858243:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e51 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941533.858244:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941533.858257:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941533.858259:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:0.0:1773941533.858264:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[180, 181] 00010000:00010000:1.0:1773941533.858267:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e51 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941533.858270:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941533.858278:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [287:287:256:4294967295] 192.168.204.143@tcp LPNI seq info [182:182:8:4294967295] 00010000:00000001:1.0:1773941533.858282:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:0.0:1773941533.858286:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941533.858288:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e51 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941533.858300:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941533.858302:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941533.858306:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:0.0:1773941533.858308:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941533.858310:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e51 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:0.0:1773941533.858315:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b91e1e300. 00010000:00000001:1.0:1773941533.858322:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000800:00000200:0.0:1773941533.858326:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941533.858327:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941533.858334:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000800:00000200:0.0:1773941533.858339:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00010000:1.0:1773941533.858340:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e51 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941533.858345:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b91e1e300 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:1.0:1773941533.858356:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138ec00 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941533.858367:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:0.0:1773941533.858367:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00010000:1.0:1773941533.858389:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e51 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941533.858393:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941533.858399:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294974221 00010000:00000001:1.0:1773941533.858406:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:0.0:1773941533.858408:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294974219 00010000:00000001:1.0:1773941533.858409:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941533.858414:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941533.858417:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ec00 refcount=5 00000100:00000001:0.0:1773941533.858419:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294974221 00000020:00000001:1.0:1773941533.858421:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941533.858427:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00000100:00000001:0.0:1773941533.858428:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294974219 00010000:00000001:1.0:1773941533.858431:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000001:0.0:1773941533.858435:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00010000:1.0:1773941533.858437:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e51 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941533.858439:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941533.858446:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941533.858453:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941533.858454:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941533.858459:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e51 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941533.858464:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:01000000:1.0:1773941533.858466:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:0.0:1773941533.858469:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 10000000:00000040:1.0:1773941533.858471:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000001:0.0:1773941533.858474:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000001:1.0:1773941533.858475:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941533.858477:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00010000:1.0:1773941533.858481:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:0.0:1773941533.858482:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941533.858486:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:0.0:1773941533.858489:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941533.858492:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000800:00000200:2.0:1773941533.858495:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941533.858497:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000800:00000010:2.0:1773941533.858503:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b91e1e300. 00010000:00000001:1.0:1773941533.858504:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941533.858508:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941533.858511:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:1.0:1773941533.858513:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138ec00 with handle 0xa33a5be14eae1e51 from hash 00010000:00000001:1.0:1773941533.858519:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:2.0:1773941533.858539:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00010000:00000001:1.0:1773941533.858540:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941533.858543:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941533.858544:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000400:00000200:2.0:1773941533.858545:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c880 00010000:00000001:1.0:1773941533.858546:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000010:2.0:1773941533.858550:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c880. 00010000:00000001:1.0:1773941533.858551:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941533.858555:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941533.858556:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941533.858561:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241167360/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941533.858563:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941533.858566:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941533.858570:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941533.858575:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e51 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941533.858576:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941533.858582:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241167360/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941543 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941533.858584:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941533.858587:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941533.858589:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941533.858592:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:2.0:1773941533.858593:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941533.858594:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941533.858597:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1773941533.858598:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941533.858601:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941533.858603:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e51 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941533.858608:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec600 count: 0 00010000:00000001:1.0:1773941533.858619:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941533.858620:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941533.858622:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941533.858626:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60e00 x1860112241167232/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941549 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941533.858632:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941533.858633:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941533.858636:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734000. 02000000:00000001:1.0:1773941533.858638:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941533.858640:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941533.858642:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941533.858643:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941533.858645:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941533.858648:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941533.858649:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941533.858667:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60e00. 00000100:00000001:1.0:1773941533.858671:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941533.858672:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941533.858674:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941533.858681:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941533.858684:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941533.858688:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0a00. 00000020:00000010:1.0:1773941533.858691:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941533.858697:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941533.858701:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941533.858705:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941533.858710:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941533.858714:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941533.858719:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941533.858724:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 10000000:00000040:1.0:1773941533.858728:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941533.858730:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941533.858734:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941533.858737:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941533.858740:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941533.858744:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941533.858746:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941533.858795:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941533.858798:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941533.858801:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941533.858805:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0a00. 10000000:01000000:1.0:1773941533.858808:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941533.858812:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941533.858814:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941533.858816:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941533.858818:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941533.858820:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941533.858823:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941533.860229:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941533.860236:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941533.860238:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941533.860239:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941533.860243:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941533.860246:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941533.860251:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941533.860253:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941533.860256:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941533.860258:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608c00 00000400:00000200:2.0:1773941533.860261:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941533.860271:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1385 [1] + 0 00000800:00000001:2.0:1773941533.860275:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941533.860286:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941533.860288:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941533.860292:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941533.860294:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941533.860298:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241167360/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941543 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941533.860306:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63800 x1860112241167360/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941543 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941533.860321:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941533.860343:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941533.860348:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941533.860391:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941533.860396:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941533.860401:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941533.860402:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941533.860405:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941533.860420:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941533.860423:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941533.860428:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941533.860438:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241167360/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941543 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941533.860456:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241167360/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941543 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941533.860469:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941533.860472:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941533.860475:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941533.860479:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941533.860481:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941533.860484:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941533.860488:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941533.860493:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=112 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941533.860499:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941533.860503:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941533.860506:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941533.860510:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941533.860513:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941533.860517:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941533.860524:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241167360/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941543 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941533.860537:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941533.860540:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941533.860542:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941533.860545:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941533.860550:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941533.860552:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941533.860553:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941533.860556:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941533.860558:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941533.860561:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941533.860563:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941533.860576:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941533 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941533.860579:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941533.860586:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941533.860588:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941533.860590:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941533.860592:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941533.860594:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941533.860598:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941533.860600:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941533.860603:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941533.860606:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941533.860609:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941533.860612:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941533.860888:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941533.860893:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941533.860898:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241167360/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941543 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941533.860903:0:12253:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:0.0:1773941533.860915:0:12253:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:1.0:1773941533.860919:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241167360:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00100000:0.0:1773941533.860938:0:12253:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941533.860948:0:12253:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:1.0:1773941533.860949:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:0.0:1773941533.860955:0:12253:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:1.0:1773941533.860956:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000001:00000001:0.0:1773941533.860966:0:12253:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000040:1.0:1773941533.860967:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241167360/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941543 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:0.0:1773941533.860975:0:12253:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:1.0:1773941533.860985:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941533.860988:0:12253:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941533.860990:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941533.860995:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0c00. 00000001:00000001:0.0:1773941533.860999:0:12253:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:1.0:1773941533.861002:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941533.861006:0:12253:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:1.0:1773941533.861007:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941533.861015:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941533.861015:0:12253:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941533.861025:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000001:0.0:1773941533.861028:0:12253:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941533.861031:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734c00. 00000100:00000001:0.0:1773941533.861037:0:12253:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:1.0:1773941533.861038:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941533.861044:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941533.861046:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00080000:0.0:1773941533.861050:0:12253:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941533.861051:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941533.861055:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941533.861079:0:12253:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941533.861080:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941533.861087:0:12253:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941533.861091:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941533.861093:0:12253:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941533.861097:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941533.861101:0:12253:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941533.861106:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941533.861109:0:12253:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:1.0:1773941533.861112:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:0.0:1773941533.861115:0:12253:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941533.861121:0:12253:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941533.861170:0:12253:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941533.861189:0:12253:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941533.861190:0:12253:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941533.861195:0:12253:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941533.861198:0:12253:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941533.861200:0:12253:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941533.861201:0:12253:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941533.861203:0:12253:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941533.861204:0:12253:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941533.863752:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec600. 00010000:00000010:1.1:1773941533.863761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138ec00. 00000020:00000001:3.0:1773941534.559854:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941534.559885:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941534.559891:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941534.623913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1773941534.623923:0:37:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1773941534.623933:0:37:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b850ab800. 00000020:00000040:0.0:1773941534.623938:0:37:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941534.623941:0:37:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941534.623944:0:37:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941534.623948:0:37:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941534.623957:0:37:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b91e1e300. 00000100:00000001:0.0:1773941534.623963:0:37:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941534.623984:0:37:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS req@ffff8b5b850ab800 x1860112241167488/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941534.623992:0:37:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941534.623995:0:37:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941534.624000:0:37:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043b1e523] to pc [ptlrpcd_00_00+0] req@ffff8b5b850ab800 x1860112241167488/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941534.624031:0:37:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941534.624035:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (310) 00000100:00000001:0.0:1773941534.624048:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1773941534.624063:0:2353:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1773941534.624064:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941534.624067:0:2352:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941534.624071:0:2352:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941534.624071:0:2353:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941534.624075:0:2353:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941534.624078:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941534.624080:0:2353:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941534.624082:0:2353:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941534.624100:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b850ab800 x1860112241167488/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941534.624106:0:2353:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941534.624107:0:2353:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941534.624135:0:2353:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941534.624137:0:2353:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941534.624139:0:2353:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941534.624142:0:2353:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2353:1860112241167488:192.168.204.143@tcp:400:kworker.0 00000100:00000001:0.0:1773941534.624145:0:2353:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941534.624150:0:2353:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941534.624152:0:2353:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941534.624154:0:2353:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941534.624158:0:2353:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b845c4a00. 02000000:00000001:0.0:1773941534.624171:0:2353:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941534.624177:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941534.624181:0:2353:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941534.624187:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941534.624192:0:2353:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241167488, portal 25 00000100:00000001:0.0:1773941534.624194:0:2353:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941534.624195:0:2353:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615823923197952 : -128249786353664 : ffff8b5b850ab800) 00000100:00000040:0.0:1773941534.624199:0:2353:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b850ab800 x1860112241167488/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941534.624205:0:2353:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941534.624207:0:2353:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941534.624209:0:2353:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c908. 00000100:00000200:0.0:1773941534.624212:0:2353:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241167488, offset 0 00000400:00000200:0.0:1773941534.624216:0:2353:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941534.624223:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941534.624229:0:2353:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941534.624233:0:2353:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941534.624236:0:2353:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941534.624239:0:2353:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941534.624264:0:2353:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941534.624271:0:2353:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[182, 181] 00000400:00000200:0.0:1773941534.624274:0:2353:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:0.0:1773941534.624277:0:2353:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [288:288:256:4294967295] 192.168.204.153@tcp LPNI seq info [183:183:8:4294967295] 00000400:00000200:0.0:1773941534.624281:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:0.0:1773941534.624289:0:2353:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:0.0:1773941534.624294:0:2353:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:0.0:1773941534.624298:0:2353:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00000800:00000200:0.0:1773941534.624304:0:2353:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:0.0:1773941534.624310:0:2353:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:0.0:1773941534.624330:0:2353:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1773941534.624354:0:2353:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941534.624359:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941534.624362:0:2353:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294974987 00000100:00000001:0.0:1773941534.624369:0:2353:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294974221 00000100:00000001:0.0:1773941534.624373:0:2353:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294974987 00000100:00000001:0.0:1773941534.624377:0:2353:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294974221 00000100:00000001:0.0:1773941534.624380:0:2353:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941534.624383:0:2353:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941534.624386:0:2353:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941534.624389:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941534.624495:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941534.624499:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000400:00000200:2.0:1773941534.624504:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941534.624508:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941534.624513:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c908 00000400:00000010:2.0:1773941534.624514:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c908. 00000100:00000001:2.0:1773941534.624518:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941534.624521:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b850ab800 x1860112241167488/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941534.624527:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941534.624530:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b850ab800 x1860112241167488/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941550 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941534.624535:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941534.624536:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941534.625791:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.625804:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941534.625807:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.625810:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941534.625816:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941534.625832:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941534.625847:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941534.625851:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941534.625855:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941534.625860:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608c80 00000400:00000200:2.0:1773941534.625864:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941534.625870:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x138d [1] + 224 00000800:00000001:2.0:1773941534.625877:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.625887:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941534.625889:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941534.625893:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941534.625898:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941534.625903:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b850ab800 x1860112241167488/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941550 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941534.625914:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b850ab800 x1860112241167488/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941550 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941534.625935:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941534.625943:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.625948:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941534.625983:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941534.625985:0:2353:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941534.625989:0:2353:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941534.625991:0:2353:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941534.625993:0:2353:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941534.625996:0:2353:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941534.625998:0:2353:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941534.626000:0:2353:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941534.626004:0:2353:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b850ab800 x1860112241167488/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941550 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941534.626012:0:2353:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b850ab800 x1860112241167488/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941550 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941534.626033:0:2353:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941534.626035:0:2353:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941534.626036:0:2353:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941534.626039:0:2353:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941534.626041:0:2353:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941534.626043:0:2353:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941534.626045:0:2353:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941534.626050:0:2353:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=113 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941534.626052:0:2353:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941534.626055:0:2353:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941534.626056:0:2353:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941534.626059:0:2353:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941534.626061:0:2353:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941534.626064:0:2353:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941534.626067:0:2353:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941534.626070:0:2353:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941534.626071:0:2353:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941534.626073:0:2353:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941534.626074:0:2353:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941534.626077:0:2353:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b850ab800 x1860112241167488/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941550 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941534.626082:0:2353:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941534.626083:0:2353:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941534.626084:0:2353:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941534.626087:0:2353:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941534.626088:0:2353:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941534.626089:0:2353:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941534.626090:0:2353:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941534.626091:0:2353:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941534.626092:0:2353:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941534.626093:0:2353:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941534.626094:0:2353:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941534.626095:0:2353:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941534.626100:0:2353:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941534.626102:0:2353:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941534.626104:0:2353:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 304 00000400:00000200:0.0:1773941534.626107:0:2353:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941534.626127:0:2353:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941534.626130:0:2353:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941534.626132:0:2353:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941534.626134:0:2353:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941534.626136:0:2353:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:0.0:1773941534.626139:0:2353:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941534.626141:0:2353:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941534.626143:0:2353:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941534.626146:0:2353:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941534.626162:0:2353:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941534.626165:0:2353:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:0.0:1773941534.626169:0:2353:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941534.626172:0:2353:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941534.626174:0:2353:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:0.0:1773941534.626176:0:2353:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:0.0:1773941534.626179:0:2353:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:0.0:1773941534.626182:0:2353:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:0.0:1773941534.626185:0:2353:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941534.626187:0:2353:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941534.626190:0:2353:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941534.626193:0:2353:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941534.626196:0:2353:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941534.626198:0:2353:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941534.626203:0:2353:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b850aad80. 00000020:00000040:0.0:1773941534.626205:0:2353:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:0.0:1773941534.626209:0:2353:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941534.626211:0:2353:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941534.626214:0:2353:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941534.626221:0:2353:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941534.626224:0:2353:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941534.626229:0:2353:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b850c0000. 00000100:00000001:0.0:1773941534.626232:0:2353:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941534.626237:0:2353:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b850aad80 x1860112241167616/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941534.626243:0:2353:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941534.626246:0:2353:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941534.626248:0:2353:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e97c40a7] to pc [ptlrpcd_rcv+-1] req@ffff8b5b850aad80 x1860112241167616/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941534.626263:0:2353:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941534.626265:0:2353:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941534.626267:0:2353:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:0.0:1773941534.626270:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b850ab800 x1860112241167488/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941550 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941534.626276:0:2353:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941534.626278:0:2353:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941534.626280:0:2353:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941534.626283:0:2353:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b850ab800 x1860112241167488/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941550 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941534.626292:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1773941534.626293:0:2353:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:1080e6b5-6a6c-4762-b0ec-c785990fec17:2353:1860112241167488:192.168.204.143@tcp:400:kworker.0 00000100:00000001:1.0:1773941534.626296:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941534.626300:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1773941534.626302:0:2353:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941534.626308:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b850aad80 x1860112241167616/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941534.626308:0:2353:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941534.626327:0:2353:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b850ab800 x1860112241167488/t0(0) o400->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941550 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941534.626332:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941534.626335:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941534.626339:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1773941534.626343:0:2353:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941534.626346:0:2353:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1773941534.626348:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941534.626351:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941534.626351:0:2353:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b845c4a00. 00000100:00100000:1.0:1773941534.626359:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b850aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241167616:192.168.204.143@tcp:250:ptlrpcd_00_01.0 02000000:00000001:0.0:1773941534.626359:0:2353:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941534.626363:0:2353:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941534.626364:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941534.626367:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941534.626368:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941534.626370:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941534.626374:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734c00. 00000020:00000040:0.0:1773941534.626375:0:2353:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941534.626377:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941534.626381:0:2353:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000200:1.0:1773941534.626383:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 02000000:00000010:0.0:1773941534.626386:0:2353:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b91e1e300. 00000400:00000010:1.0:1773941534.626387:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 02000000:00000001:0.0:1773941534.626394:0:2353:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000400:00000010:1.0:1773941534.626395:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 02000000:00000001:0.0:1773941534.626398:0:2353:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000200:1.0:1773941534.626400:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241167616, portal 25 00000100:00000010:0.0:1773941534.626402:0:2353:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b850ab800. 00000100:00000001:1.0:1773941534.626404:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941534.626407:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615823923195264 : -128249786356352 : ffff8b5b850aad80) 00000100:00000001:0.0:1773941534.626408:0:2353:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941534.626411:0:2353:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941534.626413:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b850aad80 x1860112241167616/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941534.626416:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941534.626424:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941534.626427:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000100:00000001:0.0:1773941534.626427:0:2353:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941534.626431:0:2353:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1773941534.626432:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f15d8. 00000100:00000200:1.0:1773941534.626439:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241167616, offset 0 00000100:00000001:0.0:1773941534.626439:0:2353:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941534.626444:0:2353:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941534.626446:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941534.626452:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941534.626456:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941534.626459:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941534.626461:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941534.626463:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941534.626465:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941534.626468:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[182, 183] 00000400:00000200:1.0:1773941534.626472:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:1.0:1773941534.626475:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [289:289:256:4294967295] 192.168.204.143@tcp LPNI seq info [184:184:8:4294967295] 00000400:00000200:1.0:1773941534.626480:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941534.626494:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:1.0:1773941534.626499:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:1.0:1773941534.626504:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b400. 00000800:00000200:1.0:1773941534.626510:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000800:00000200:1.0:1773941534.626517:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:1.0:1773941534.626521:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b400 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:1.0:1773941534.626534:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941534.626538:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941534.626539:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294974989 00000100:00000001:1.0:1773941534.626543:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294974987 00000100:00000001:1.0:1773941534.626547:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294974989 00000100:00000001:1.0:1773941534.626549:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294974987 00000100:00000001:1.0:1773941534.626550:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941534.626552:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.626554:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.626556:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941534.626560:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941534.626561:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1773941534.626563:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941534.626564:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941534.626566:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.626567:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941534.626622:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941534.626627:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b400. 00000400:00000200:2.0:1773941534.626631:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941534.626636:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000400:00000200:2.0:1773941534.626639:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f15d8 00000400:00000010:2.0:1773941534.626642:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f15d8. 00000100:00000001:2.0:1773941534.626645:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941534.626649:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b850aad80 x1860112241167616/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941534.626657:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941534.626660:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b850aad80 x1860112241167616/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941544 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941534.626667:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941534.626674:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941534.628385:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.628392:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941534.628394:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.628396:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941534.628399:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941534.628420:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941534.628425:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941534.628428:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941534.628431:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941534.628434:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608d00 00000400:00000200:2.0:1773941534.628437:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941534.628441:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x1395 [1] + 0 00000800:00000001:2.0:1773941534.628445:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.628450:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941534.628452:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941534.628455:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941534.628458:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941534.628462:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b850aad80 x1860112241167616/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941544 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941534.628470:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b850aad80 x1860112241167616/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941544 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941534.628482:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941534.628486:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.628488:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941534.628520:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941534.628523:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941534.628528:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941534.628529:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941534.628532:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941534.628536:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941534.628538:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941534.628541:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941534.628546:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b850aad80 x1860112241167616/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941544 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941534.628553:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b850aad80 x1860112241167616/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941544 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941534.628565:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941534.628567:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941534.628568:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941534.628570:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941534.628571:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941534.628573:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941534.628575:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941534.628577:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=113 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941534.628580:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941534.628581:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941534.628583:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941534.628585:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941534.628586:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.628588:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941534.628591:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b850aad80 x1860112241167616/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941544 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941534.628596:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941534.628597:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941534.628599:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941534.628600:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1773941534.628605:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:1.0:1773941534.628606:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941534.628608:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941534.628609:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941534.628611:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941534.628612:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941534.628614:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:1.0:1773941534.628617:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941534 v=0 (0 0 0 0) 00000100:00001000:1.0:1773941534.628619:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:1.0:1773941534.628624:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:1.0:1773941534.628625:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941534.628626:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:1.0:1773941534.628628:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.628632:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941534.628635:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:1.0:1773941534.628636:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:1.0:1773941534.628638:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:1.0:1773941534.628641:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:1.0:1773941534.628643:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:1.0:1773941534.628644:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:0.0:1773941534.628896:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941534.628901:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941534.628909:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b850aad80 x1860112241167616/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941544 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941534.628917:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b850aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241167616:192.168.204.143@tcp:250:ptlrpcd_00_01.0 00000100:00000001:0.0:1773941534.628922:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941534.628924:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941534.628926:0:12254:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:0.0:1773941534.628927:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b850aad80 x1860112241167616/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941544 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941534.628935:0:12254:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:1.0:1773941534.628940:0:12254:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941534.628943:0:12254:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:0.0:1773941534.628943:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 10000000:01000000:1.0:1773941534.628946:0:12254:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:0.0:1773941534.628948:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941534.628949:0:12254:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:0.0:1773941534.628953:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734c00. 00000001:00000001:1.0:1773941534.628954:0:12254:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941534.628958:0:12254:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941534.628960:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941534.628961:0:12254:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:0.0:1773941534.628964:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:1.0:1773941534.628976:0:12254:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:0.0:1773941534.628976:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:1.0:1773941534.628978:0:12254:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941534.628983:0:12254:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941534.628983:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941534.628987:0:12254:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:0.0:1773941534.628988:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b850c0000. 00000100:00080000:1.0:1773941534.628991:0:12254:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:0.0:1773941534.628994:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941534.628995:0:12254:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:0.0:1773941534.628999:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941534.629000:0:12254:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941534.629005:0:12254:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941534.629006:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b850aad80. 00000100:00080000:1.0:1773941534.629013:0:12254:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941534.629014:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941534.629018:0:12254:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941534.629019:0:12254:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941534.629021:0:12254:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00000001:0.0:1773941534.629025:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941534.629030:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941534.629034:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941534.629036:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941534.629039:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941534.629051:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941534.629053:0:12254:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941534.629057:0:12254:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.629060:0:12254:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941534.629065:0:12254:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941534.629069:0:12254:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941534.629073:0:12254:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941534.629085:0:12254:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941534.629089:0:12254:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941534.629091:0:12254:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941534.879878:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941534.879887:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941534.879893:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941534.879896:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941534.879897:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941534.879899:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941534.879901:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941534.879903:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941534.879906:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941534.879909:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941534.879911:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941534.879923:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941534.879928:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941534.879933:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941534.879936:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941534.879941:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941534.879949:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941534.879956:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941534.879962:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941534.879965:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941534.879973:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec600. 00010000:00000001:1.0:1773941534.879980:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941534.879984:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138ec00. 00000020:00000001:1.0:1773941534.879988:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941534.879991:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138ec00 with handle 0xa33a5be14eae1e58 to hash 00000020:00000001:1.0:1773941534.879994:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941534.879996:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941534.879999:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941534.880003:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941534.880007:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941534.880012:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e58 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941534.880045:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e58 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941534.880059:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e58 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941534.880066:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941534.880068:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941534.880075:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941534.880078:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941534.880081:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941534.880082:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000001:1.0:1773941534.880088:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941534.880092:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.880095:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941534.880097:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941534.880124:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241167744/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941534.880135:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941534.880137:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941534.880141:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941534.880144:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941534.880147:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941534.880151:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241167744:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941534.880156:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941534.880161:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941534.880164:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941534.880166:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941534.880171:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0734000. 02000000:00000001:1.0:1773941534.880174:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941534.880182:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941534.880193:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941534.880201:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941534.880208:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241167744, portal 25 00000100:00000001:1.0:1773941534.880210:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941534.880212:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:1.0:1773941534.880218:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241167744/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941550 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941534.880227:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941534.880230:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941534.880233:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1a18. 00000100:00000200:1.0:1773941534.880237:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241167744, offset 0 00000400:00000200:1.0:1773941534.880244:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941534.880254:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941534.880261:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941534.880266:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941534.880271:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941534.880275:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941534.880278:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941534.880282:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[184, 183] 00000400:00000200:1.0:1773941534.880285:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941534.880290:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [290:290:256:4294967295] 192.168.204.153@tcp LPNI seq info [185:185:8:4294967295] 00000400:00000200:1.0:1773941534.880295:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941534.880305:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941534.880311:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941534.880317:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024bf00. 00000800:00000200:1.0:1773941534.880324:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941534.880332:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941534.880336:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024bf00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941534.880356:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941534.880360:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941534.880363:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294975243 00000100:00000001:1.0:1773941534.880375:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294974989 00000100:00000001:1.0:1773941534.880380:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294975243 00000100:00000001:1.0:1773941534.880382:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294974989 00000100:00000001:1.0:1773941534.880387:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941534.880390:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.880392:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941534.880393:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941534.880395:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941534.880397:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941534.880399:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941534.880493:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941534.880498:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024bf00. 00000400:00000200:2.0:1773941534.880505:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941534.880523:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941534.880530:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1a18 00000400:00000010:2.0:1773941534.880533:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1a18. 00000100:00000001:2.0:1773941534.880538:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941534.880544:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241167744/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941550 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941534.880563:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941534.880567:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d63800 x1860112241167744/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941550 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941534.880576:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941534.880578:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941534.881547:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.881560:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941534.881563:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.881566:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941534.881571:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941534.881576:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941534.881583:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941534.881587:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941534.881591:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941534.881595:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608d80 00000400:00000200:2.0:1773941534.881599:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941534.881604:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x139d [1] + 224 00000800:00000001:2.0:1773941534.881611:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.881619:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941534.881622:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941534.881626:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941534.881630:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941534.881635:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241167744/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941550 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941534.881647:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d63800 x1860112241167744/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941550 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941534.881665:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941534.881673:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.881677:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941534.881692:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941534.881697:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941534.881699:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941534.881701:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941534.881736:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941534.881737:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941534.881739:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941534.881744:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241167744/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941550 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941534.881752:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241167744/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941550 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941534.881757:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941534.881758:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941534.881760:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941534.881762:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941534.881764:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941534.881765:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941534.881767:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941534.881772:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=113 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941534.881774:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941534.881776:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941534.881779:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941534.881781:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941534.881790:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941534.881792:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941534.881795:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941534.881798:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941534.881801:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941534.881803:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941534.881804:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941534.881806:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941534.881808:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d63800 x1860112241167744/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941550 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941534.881813:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941534.881814:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941534.881816:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941534.881817:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941534.881819:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941534.881820:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.881822:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941534.881822:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941534.881823:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941534.881824:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.881825:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941534.881826:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941534.881828:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941534.881830:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941534.881832:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 305 00000400:00000200:1.0:1773941534.881836:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941534.881839:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941534.881842:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941534.881844:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941534.881846:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941534.881848:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941534.881850:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941534.881855:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941534.881856:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941534.881858:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941534.881860:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941534.881862:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941534.881864:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941534.881865:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941534.881866:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941534.881868:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941534.881870:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941534.881872:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941534.881874:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941534.881876:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941534.881878:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941534.881880:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.881881:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941534.881882:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941534.881885:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60e00. 00000020:00000040:1.0:1773941534.881887:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941534.881889:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941534.881890:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.881892:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941534.881893:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941534.881895:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941534.881898:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737000. 00000100:00000001:1.0:1773941534.881901:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941534.881905:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60e00 x1860112241167872/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941534.881911:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941534.881914:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941534.881917:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5e4d003] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60e00 x1860112241167872/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941534.881943:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.881946:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941534.881947:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941534.881949:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941534.881951:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941534.881952:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941534.881953:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941534.881954:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941534.881956:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941534.881959:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241167744/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941550 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941534.881964:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941534.881966:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941534.881968:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941534.881970:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941534.881973:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241167744/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941550 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941534.881980:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241167744:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941534.881985:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941534.881987:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941534.881989:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941534.881990:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941534.881992:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1773941534.881994:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241167744/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941550 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941534.882001:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941534.882003:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941534.882006:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:0.0:1773941534.882006:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941534.882013:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941534.882015:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941534.882022:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000040:0.0:1773941534.882023:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60e00 x1860112241167872/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941534.882032:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941534.882041:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1773941534.882041:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941534.882047:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941534.882048:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ec00 refcount=4 00000020:00000001:1.0:1773941534.882056:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 02000000:00000001:0.0:1773941534.882057:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:1.0:1773941534.882061:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 02000000:00000001:0.0:1773941534.882063:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00010000:1.0:1773941534.882066:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e58 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941534.882066:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941534.882078:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241167872:192.168.204.143@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941534.882088:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941534.882089:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00010000:1.0:1773941534.882098:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e58 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941534.882099:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941534.882119:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941534.882126:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941534.882133:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000010:0.0:1773941534.882133:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0000. 00010000:00010000:1.0:1773941534.882137:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e58 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:0.0:1773941534.882137:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941534.882151:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941534.882154:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:0.0:1773941534.882154:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941534.882157:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e58 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941534.882159:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00000001:1.0:1773941534.882170:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000010:0.0:1773941534.882172:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00010000:00000001:1.0:1773941534.882173:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941534.882178:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000200:0.0:1773941534.882179:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241167872, portal 25 00010000:00010000:1.0:1773941534.882181:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e58 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941534.882185:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941534.882189:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00010000:00010000:1.0:1773941534.882201:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138ec00 already unused, calling callback (ffffffffc1482c20) 00000100:00000040:0.0:1773941534.882205:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60e00 x1860112241167872/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941534.882207:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941534.882213:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e58 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941534.882235:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941534.882241:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941534.882245:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c440. 00010000:00000001:1.0:1773941534.882251:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941534.882253:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941534.882254:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941534.882258:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ec00 refcount=5 00000100:00000200:0.0:1773941534.882262:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241167872, offset 0 00000020:00000001:1.0:1773941534.882263:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941534.882274:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00000400:00000200:0.0:1773941534.882277:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941534.882278:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941534.882287:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e58 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941534.882300:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:1.0:1773941534.882301:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941534.882304:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e58 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941534.882309:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941534.882311:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941534.882312:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941534.882314:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941534.882316:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941534.882319:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941534.882321:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941534.882322:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941534.882325:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941534.882326:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000020:00000040:1.0:1773941534.882328:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138ec00 with handle 0xa33a5be14eae1e58 from hash 00010000:00000001:1.0:1773941534.882332:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941534.882333:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941534.882335:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941534.882336:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941534.882340:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941534.882342:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941534.882343:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:0.0:1773941534.882347:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941534.882348:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941534.882352:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941534.882354:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941534.882359:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000001:1.0:1773941534.882362:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941534.882365:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:1.0:1773941534.882368:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e58 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941534.882374:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[184, 185] 00010000:00000001:1.0:1773941534.882395:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941534.882396:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941534.882398:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941534.882400:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941534.882404:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000400:00000200:0.0:1773941534.882404:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [291:291:256:4294967295] 192.168.204.143@tcp LPNI seq info [186:186:8:4294967295] 00010000:00000001:1.0:1773941534.882410:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941534.882417:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941534.882419:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941534.882423:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941534.882426:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e58 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941534.882436:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec600 count: 0 00000400:00000200:0.0:1773941534.882441:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00000800:00000200:0.0:1773941534.882449:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941534.882454:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00000800:00000200:0.0:1773941534.882461:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941534.882462:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:0.0:1773941534.882469:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941534.882470:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000200:0.0:1773941534.882475:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:1.0:1773941534.882479:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941534.882489:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941534.882490:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241167744/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941550 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941534.882499:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941534.882504:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294975245 00000100:00000001:0.0:1773941534.882515:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294975243 00000100:00000001:0.0:1773941534.882520:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294975245 00000100:00000001:0.0:1773941534.882525:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294975243 00000100:00000001:1.0:1773941534.882530:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941534.882531:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941534.882534:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1773941534.882534:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941534.882542:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0734000. 00000100:00000001:0.0:1773941534.882543:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941534.882556:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1773941534.882557:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941534.882564:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941534.882566:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941534.882569:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941534.882569:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:2.0:1773941534.882573:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1773941534.882574:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000010:1.0:1773941534.882575:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941534.882579:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941534.882582:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941534.882588:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:0.0:1773941534.882591:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:2.0:1773941534.882592:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000100:00000001:1.0:1773941534.882592:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941534.882596:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941534.882598:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941534.882600:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941534.882601:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000001:1.0:1773941534.882602:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941534.882612:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941534.882613:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000400:00000200:2.0:1773941534.882620:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 10000000:00000001:1.0:1773941534.882627:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000400:00000200:2.0:1773941534.882629:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c440 00000400:00000010:2.0:1773941534.882631:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c440. 00000100:00000001:2.0:1773941534.882635:0:2347:0:(events.c:34:request_out_callback()) Process entered 10000000:00000010:1.0:1773941534.882640:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0080. 00000100:00000200:2.0:1773941534.882643:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60e00 x1860112241167872/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941544 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941534.882646:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941534.882653:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000100:00000001:2.0:1773941534.882655:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000040:00000010:1.0:1773941534.882657:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000100:00000040:2.0:1773941534.882660:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60e00 x1860112241167872/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941544 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941534.882663:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000100:00000001:2.0:1773941534.882671:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941534.882671:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 00000100:00000001:2.0:1773941534.882674:0:2347:0:(events.c:68:request_out_callback()) Process leaving 80000000:00000010:1.0:1773941534.882677:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941534.882682:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941534.882686:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941534.882691:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941534.882693:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941534.882697:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941534.882699:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941534.882701:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941534.882736:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941534.882743:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941534.882747:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941534.882750:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941534.882753:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941534.882755:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0080. 10000000:01000000:1.0:1773941534.882758:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941534.882760:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941534.882762:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941534.882764:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941534.882765:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941534.882767:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941534.882769:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941534.884269:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.884278:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941534.884280:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.884281:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941534.884285:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941534.884289:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941534.884294:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941534.884297:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941534.884304:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941534.884306:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608e00 00000400:00000200:2.0:1773941534.884309:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941534.884312:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x13a5 [1] + 0 00000800:00000001:2.0:1773941534.884317:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.884323:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941534.884324:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941534.884327:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941534.884330:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941534.884334:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60e00 x1860112241167872/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941544 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941534.884342:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60e00 x1860112241167872/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941544 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941534.884364:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941534.884369:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941534.884371:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941534.884401:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941534.884405:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941534.884410:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941534.884411:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941534.884414:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941534.884418:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941534.884420:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941534.884422:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941534.884428:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60e00 x1860112241167872/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941544 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941534.884437:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60e00 x1860112241167872/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941544 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941534.884443:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941534.884444:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941534.884446:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941534.884448:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941534.884450:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941534.884451:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941534.884453:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941534.884458:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=113 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941534.884460:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941534.884463:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941534.884464:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941534.884466:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941534.884468:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941534.884470:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941534.884474:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60e00 x1860112241167872/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941544 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941534.884479:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941534.884481:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941534.884483:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941534.884492:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941534.884498:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941534.884500:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941534.884502:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941534.884504:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941534.884506:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941534.884508:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941534.884510:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941534.884513:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941534 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941534.884515:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941534.884522:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941534.884524:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941534.884526:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941534.884528:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941534.884530:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941534.884533:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941534.884535:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941534.884537:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941534.884540:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941534.884543:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941534.884545:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941534.884795:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.884801:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941534.884809:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60e00 x1860112241167872/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941544 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941534.884818:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241167872:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941534.884818:0:12255:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:0.0:1773941534.884835:0:12255:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:1.0:1773941534.884841:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941534.884844:0:12255:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941534.884845:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941534.884851:0:12255:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000040:1.0:1773941534.884853:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60e00 x1860112241167872/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941544 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:01000000:0.0:1773941534.884857:0:12255:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:1.0:1773941534.884868:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941534.884869:0:12255:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:1.0:1773941534.884870:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941534.884876:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0000. 00000001:00000001:0.0:1773941534.884877:0:12255:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:1.0:1773941534.884880:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941534.884883:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941534.884884:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941534.884885:0:12255:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941534.884887:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941534.884891:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737000. 00000001:00000001:0.0:1773941534.884893:0:12255:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:1.0:1773941534.884895:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:0.0:1773941534.884897:0:12255:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:1.0:1773941534.884898:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941534.884903:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60e00. 00000001:00000001:0.0:1773941534.884904:0:12255:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941534.884915:0:12255:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.884917:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941534.884924:0:12255:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941534.884925:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941534.884931:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941534.884935:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941534.884937:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941534.884939:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941534.884940:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941534.884947:0:12255:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941534.884950:0:12255:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941534.884952:0:12255:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941534.884954:0:12255:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941534.884956:0:12255:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941534.884958:0:12255:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941534.884959:0:12255:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941534.884961:0:12255:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941534.884974:0:12255:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941534.884975:0:12255:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941534.884976:0:12255:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941534.884980:0:12255:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941534.884985:0:12255:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941534.884988:0:12255:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941534.884991:0:12255:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941534.884994:0:12255:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941534.884996:0:12255:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941534.887737:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec600. 00010000:00000010:1.1:1773941534.887744:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138ec00. 00000020:00000001:3.0:1773941535.583889:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941535.583898:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941535.583905:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941536.415873:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941536.415883:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941536.415889:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941536.415892:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941536.415894:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941536.415895:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941536.415899:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941536.415901:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941536.415904:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941536.415906:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941536.415908:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941536.415925:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60e00. 00000020:00000040:1.0:1773941536.415934:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941536.415940:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941536.415969:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941536.415978:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941536.415990:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941536.416002:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941536.416009:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941536.416012:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941536.416021:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec600. 00010000:00000001:1.0:1773941536.416028:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941536.416032:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138ec00. 00000020:00000001:1.0:1773941536.416037:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941536.416040:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138ec00 with handle 0xa33a5be14eae1e5f to hash 00000020:00000001:1.0:1773941536.416042:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941536.416044:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941536.416047:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941536.416051:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941536.416055:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941536.416059:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e5f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941536.416070:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e5f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941536.416099:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e5f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941536.416109:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941536.416112:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941536.416121:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941536.416125:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941536.416129:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941536.416131:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00000100:00000001:1.0:1773941536.416138:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941536.416145:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941536.416149:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941536.416159:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941536.416175:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60e00 x1860112241168000/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941536.416189:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941536.416192:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941536.416197:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941536.416200:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941536.416203:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941536.416206:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241168000:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941536.416211:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941536.416219:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941536.416221:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941536.416224:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941536.416230:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0737000. 02000000:00000001:1.0:1773941536.416232:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941536.416242:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941536.416246:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941536.416254:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941536.416262:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241168000, portal 25 00000100:00000001:1.0:1773941536.416264:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941536.416265:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00000100:00000040:1.0:1773941536.416271:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60e00 x1860112241168000/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941552 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941536.416296:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941536.416300:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941536.416302:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1110. 00000100:00000200:1.0:1773941536.416307:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241168000, offset 0 00000400:00000200:1.0:1773941536.416314:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941536.416325:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941536.416335:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941536.416358:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941536.416364:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941536.416368:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941536.416376:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941536.416380:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[186, 185] 00000400:00000200:1.0:1773941536.416383:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941536.416387:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [292:292:256:4294967295] 192.168.204.153@tcp LPNI seq info [187:187:8:4294967295] 00000400:00000200:1.0:1773941536.416392:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941536.416404:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941536.416410:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941536.416415:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b100. 00000800:00000200:1.0:1773941536.416423:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941536.416431:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941536.416448:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b100 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941536.416465:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941536.416469:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941536.416472:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294976779 00000100:00000001:1.0:1773941536.416480:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294975245 00000100:00000001:1.0:1773941536.416485:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294976779 00000100:00000001:1.0:1773941536.416488:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294975245 00000100:00000001:1.0:1773941536.416491:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941536.416494:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941536.416496:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941536.416498:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941536.416501:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941536.416504:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941536.416508:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941536.416652:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941536.416662:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b100. 00000400:00000200:2.0:1773941536.416671:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941536.416680:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941536.416688:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1110 00000400:00000010:2.0:1773941536.416692:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1110. 00000100:00000001:2.0:1773941536.416697:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941536.416751:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60e00 x1860112241168000/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941552 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941536.416766:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941536.416771:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d60e00 x1860112241168000/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941552 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941536.416779:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.416782:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941536.417727:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941536.417741:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941536.417745:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941536.417748:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941536.417765:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941536.417772:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941536.417779:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941536.417784:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941536.417800:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941536.417805:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608e80 00000400:00000200:2.0:1773941536.417815:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941536.417834:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x13ad [1] + 224 00000800:00000001:2.0:1773941536.417843:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941536.417854:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941536.417858:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941536.417876:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941536.417882:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941536.417887:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60e00 x1860112241168000/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941552 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941536.417899:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d60e00 x1860112241168000/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941552 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941536.417924:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941536.417932:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941536.417936:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941536.417992:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941536.418001:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941536.418003:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941536.418007:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941536.418011:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941536.418014:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941536.418018:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941536.418034:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60e00 x1860112241168000/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941552 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941536.418046:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60e00 x1860112241168000/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941552 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941536.418054:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941536.418072:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941536.418075:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941536.418091:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941536.418095:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941536.418097:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941536.418100:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941536.418107:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=115 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941536.418110:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941536.418114:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941536.418118:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941536.418121:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941536.418124:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941536.418127:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941536.418132:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941536.418136:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941536.418139:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941536.418141:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941536.418143:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941536.418145:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941536.418149:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d60e00 x1860112241168000/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941552 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941536.418156:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941536.418157:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941536.418159:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941536.418161:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941536.418164:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941536.418166:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941536.418172:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941536.418174:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941536.418175:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941536.418176:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941536.418178:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941536.418180:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941536.418182:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941536.418185:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941536.418188:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 305 00000400:00000200:1.0:1773941536.418192:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941536.418196:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941536.418200:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941536.418203:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941536.418206:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941536.418208:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941536.418211:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941536.418214:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941536.418216:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941536.418218:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941536.418220:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941536.418221:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941536.418224:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941536.418225:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941536.418227:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941536.418228:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941536.418231:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941536.418234:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941536.418236:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941536.418238:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941536.418241:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941536.418244:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941536.418246:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941536.418251:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941536.418254:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941536.418256:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941536.418260:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941536.418261:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941536.418264:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941536.418266:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941536.418268:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941536.418272:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0734000. 00000100:00000001:1.0:1773941536.418275:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941536.418298:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63800 x1860112241168128/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941536.418308:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941536.418311:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941536.418317:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc928f9c] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63800 x1860112241168128/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941536.418362:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941536.418366:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941536.418368:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941536.418371:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941536.418375:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941536.418377:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941536.418379:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941536.418382:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941536.418384:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941536.418391:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60e00 x1860112241168000/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941552 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941536.418401:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941536.418405:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941536.418408:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941536.418411:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941536.418415:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1773941536.418416:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60e00 x1860112241168000/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941552 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941536.418420:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941536.418423:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941536.418430:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241168128/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941536.418441:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941536.418443:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941536.418447:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241168000:192.168.204.143@tcp:101:ll_cfg_requeue.0 02000000:00000001:0.0:1773941536.418447:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941536.418449:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941536.418452:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941536.418453:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941536.418456:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241168128:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941536.418457:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941536.418460:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941536.418461:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941536.418463:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1773941536.418464:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941536.418466:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941536.418467:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60e00 x1860112241168000/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941552 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1773941536.418469:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941536.418474:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c3000. 00000100:00000001:1.0:1773941536.418477:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941536.418477:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941536.418480:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941536.418484:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941536.418486:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:0.0:1773941536.418488:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941536.418490:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000010:0.0:1773941536.418491:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00000001:1.0:1773941536.418494:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941536.418496:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:0.0:1773941536.418497:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000020:00000040:1.0:1773941536.418499:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ec00 refcount=4 00000020:00000001:1.0:1773941536.418502:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00000100:00000200:0.0:1773941536.418502:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241168128, portal 25 00000100:00000001:0.0:1773941536.418505:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941536.418506:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00000100:00000001:0.0:1773941536.418506:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00010000:00010000:1.0:1773941536.418512:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e5f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941536.418513:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241168128/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941536.418523:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941536.418523:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941536.418527:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941536.418529:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e5f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941536.418530:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6cb28. 00000100:00000200:0.0:1773941536.418534:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241168128, offset 0 00010000:00000001:1.0:1773941536.418539:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:0.0:1773941536.418540:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941536.418543:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e5f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941536.418546:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941536.418551:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:1.0:1773941536.418552:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941536.418555:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941536.418559:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:0.0:1773941536.418559:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941536.418562:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00010000:1.0:1773941536.418563:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e5f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941536.418566:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941536.418571:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[186, 187] 00010000:00000001:1.0:1773941536.418573:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941536.418576:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941536.418576:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:1.0:1773941536.418580:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:0.0:1773941536.418581:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [293:293:256:4294967295] 192.168.204.143@tcp LPNI seq info [188:188:8:4294967295] 00010000:00010000:1.0:1773941536.418584:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e5f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941536.418586:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941536.418594:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138ec00 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:0.0:1773941536.418596:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 10000000:00000001:1.0:1773941536.418598:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941536.418601:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e5f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941536.418602:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941536.418608:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00010000:00000001:1.0:1773941536.418611:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941536.418612:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941536.418615:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:0.0:1773941536.418615:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00000020:00000040:1.0:1773941536.418617:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ec00 refcount=5 00000800:00000200:0.0:1773941536.418624:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000800:00000200:0.0:1773941536.418628:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00000020:00000001:1.0:1773941536.418631:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941536.418634:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941536.418639:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941536.418643:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e5f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941536.418644:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941536.418649:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:1.0:1773941536.418652:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941536.418655:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e5f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941536.418657:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294976781 00000100:00000001:0.0:1773941536.418663:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294976779 10000000:01000000:1.0:1773941536.418664:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941536.418668:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000001:0.0:1773941536.418669:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294976781 10000000:00000001:1.0:1773941536.418671:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941536.418674:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294976779 00010000:00010000:1.0:1773941536.418675:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941536.418678:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:0.0:1773941536.418680:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941536.418684:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941536.418686:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:0.0:1773941536.418689:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941536.418690:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941536.418693:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:0.0:1773941536.418694:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941536.418695:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941536.418699:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138ec00 with handle 0xa33a5be14eae1e5f from hash 00010000:00000001:1.0:1773941536.418702:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941536.418743:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941536.418745:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941536.418746:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941536.418748:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:0.0:1773941536.418748:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:1.0:1773941536.418750:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:0.0:1773941536.418751:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941536.418753:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941536.418754:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:0.0:1773941536.418755:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941536.418757:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941536.418758:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941536.418759:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941536.418762:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941536.418766:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941536.418772:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00010000:1.0:1773941536.418772:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e5f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:2.0:1773941536.418781:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00010000:00000001:1.0:1773941536.418791:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941536.418792:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:1.0:1773941536.418795:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941536.418800:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000400:00000200:2.0:1773941536.418803:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00010000:00000001:1.0:1773941536.418805:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941536.418809:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941536.418811:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6cb28 00010000:00000001:1.0:1773941536.418813:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:2.0:1773941536.418816:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6cb28. 00010000:00000001:1.0:1773941536.418818:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941536.418823:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e5f lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941536.418824:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941536.418835:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241168128/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000040:1.0:1773941536.418841:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec600 count: 0 00000100:00000001:2.0:1773941536.418853:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941536.418858:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241168128/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941536.418864:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941536.418869:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941536.418871:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941536.418881:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941536.418882:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1773941536.418895:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60e00 x1860112241168000/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941552 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941536.418909:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941536.418914:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941536.418923:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0737000. 02000000:00000001:1.0:1773941536.418927:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941536.418929:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941536.418931:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941536.418933:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941536.418936:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941536.418939:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941536.418941:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941536.418943:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60e00. 00000100:00000001:1.0:1773941536.418946:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941536.418947:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941536.418949:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941536.418951:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941536.418954:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941536.418958:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0080. 00000020:00000010:1.0:1773941536.418963:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941536.418968:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941536.418972:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941536.418977:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941536.418981:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941536.418986:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941536.418990:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941536.418995:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 10000000:00000040:1.0:1773941536.418999:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941536.419001:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941536.419006:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941536.419009:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941536.419012:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941536.419017:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941536.419019:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941536.419023:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941536.419026:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941536.419029:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941536.419031:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0080. 10000000:01000000:1.0:1773941536.419034:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941536.419040:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941536.419043:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941536.419045:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941536.419046:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941536.419048:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941536.419050:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941536.420560:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941536.420568:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941536.420571:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941536.420574:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941536.420578:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941536.420583:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941536.420590:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941536.420593:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941536.420595:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941536.420604:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321608f00 00000400:00000200:2.0:1773941536.420607:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941536.420610:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x13b5 [1] + 0 00000800:00000001:2.0:1773941536.420614:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941536.420620:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941536.420622:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941536.420624:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941536.420627:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941536.420631:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241168128/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941536.420639:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63800 x1860112241168128/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941536.420655:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941536.420659:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941536.420661:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941536.420695:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941536.420699:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941536.420735:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941536.420739:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941536.420743:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941536.420750:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941536.420753:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941536.420757:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941536.420766:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241168128/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941536.420779:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241168128/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941536.420786:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941536.420788:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941536.420790:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941536.420792:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941536.420794:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941536.420795:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941536.420797:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941536.420801:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=115 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941536.420804:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941536.420815:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941536.420817:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941536.420819:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941536.420821:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941536.420824:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941536.420828:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241168128/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941546 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941536.420835:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941536.420837:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941536.420839:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941536.420841:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941536.420846:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941536.420847:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941536.420849:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941536.420850:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941536.420852:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941536.420854:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941536.420866:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941536.420874:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941536 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941536.420877:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941536.420884:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941536.420886:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941536.420888:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941536.420890:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941536.420891:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941536.420895:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941536.420897:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941536.420900:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941536.420903:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941536.420906:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941536.420908:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941536.421215:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941536.421220:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941536.421227:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241168128/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941546 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941536.421239:0:12256:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:1.0:1773941536.421240:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241168128:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941536.421245:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941536.421248:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941536.421252:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241168128/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941546 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941536.421258:0:12256:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:1.0:1773941536.421261:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:0.0:1773941536.421262:0:12256:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:1.0:1773941536.421264:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1773941536.421265:0:12256:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:1.0:1773941536.421266:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c3000. 10000000:01000000:0.0:1773941536.421268:0:12256:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:1.0:1773941536.421270:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941536.421272:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941536.421272:0:12256:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000040:1.0:1773941536.421274:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941536.421276:0:12256:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:1.0:1773941536.421277:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941536.421279:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0734000. 00000001:00000001:0.0:1773941536.421280:0:12256:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941536.421282:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:0.0:1773941536.421282:0:12256:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:1.0:1773941536.421284:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:0.0:1773941536.421284:0:12256:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000010:1.0:1773941536.421286:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000001:00000001:0.0:1773941536.421287:0:12256:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941536.421289:0:12256:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941536.421290:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941536.421292:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941536.421292:0:12256:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941536.421294:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941536.421297:0:12256:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941536.421299:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941536.421300:0:12256:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941536.421301:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941536.421303:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941536.421303:0:12256:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941536.421305:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941536.421306:0:12256:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941536.421309:0:12256:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941536.421312:0:12256:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941536.421314:0:12256:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941536.421316:0:12256:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941536.421344:0:12256:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941536.421346:0:12256:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941536.421349:0:12256:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941536.421354:0:12256:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941536.421358:0:12256:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941536.421362:0:12256:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941536.421364:0:12256:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941536.421367:0:12256:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941536.421370:0:12256:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941536.424743:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec600. 00010000:00000010:1.1:1773941536.424751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138ec00. 00000020:00000001:3.0:1773941536.607938:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941536.607945:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941536.607950:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941537.631887:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941537.631892:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941537.631896:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941538.015819:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941538.015851:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941538.015856:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941538.015859:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941538.015860:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941538.015862:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941538.015865:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941538.015867:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941538.015869:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941538.015872:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941538.015873:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941538.015899:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941538.015904:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941538.015909:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941538.015913:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941538.015920:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941538.015928:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941538.015936:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941538.015941:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941538.015944:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941538.015958:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec700. 00010000:00000001:1.0:1773941538.015965:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941538.015968:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138ec00. 00000020:00000001:1.0:1773941538.015971:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941538.015974:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138ec00 with handle 0xa33a5be14eae1e66 to hash 00000020:00000001:1.0:1773941538.015976:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941538.015977:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941538.015980:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941538.015984:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941538.015988:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941538.015991:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e66 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941538.016001:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e66 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941538.016021:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e66 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941538.016026:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941538.016028:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941538.016035:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941538.016036:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941538.016039:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941538.016040:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000001:1.0:1773941538.016045:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941538.016065:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941538.016067:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941538.016069:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941538.016081:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241168256/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941538.016089:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941538.016091:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941538.016095:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941538.016097:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941538.016099:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941538.016102:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241168256:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941538.016105:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941538.016112:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941538.016114:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941538.016117:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941538.016122:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0737000. 02000000:00000001:1.0:1773941538.016123:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941538.016131:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941538.016135:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941538.016142:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941538.016149:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241168256, portal 25 00000100:00000001:1.0:1773941538.016151:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941538.016156:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:1.0:1773941538.016161:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241168256/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941554 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941538.016167:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941538.016170:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941538.016172:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1000. 00000100:00000200:1.0:1773941538.016175:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241168256, offset 0 00000400:00000200:1.0:1773941538.016181:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941538.016189:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941538.016197:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941538.016203:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941538.016207:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941538.016211:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941538.016213:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941538.016216:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[188, 187] 00000400:00000200:1.0:1773941538.016219:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941538.016223:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [294:294:256:4294967295] 192.168.204.153@tcp LPNI seq info [189:189:8:4294967295] 00000400:00000200:1.0:1773941538.016226:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941538.016236:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941538.016241:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941538.016246:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b800. 00000800:00000200:1.0:1773941538.016253:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941538.016260:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941538.016265:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b800 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941538.016282:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941538.016285:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941538.016288:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294978379 00000100:00000001:1.0:1773941538.016294:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294976781 00000100:00000001:1.0:1773941538.016299:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294978379 00000100:00000001:1.0:1773941538.016302:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294976781 00000100:00000001:1.0:1773941538.016310:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941538.016314:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941538.016317:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941538.016320:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941538.016324:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941538.016328:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941538.016332:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941538.016422:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941538.016430:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b800. 00000400:00000200:2.0:1773941538.016448:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941538.016454:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941538.016460:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1000 00000400:00000010:2.0:1773941538.016462:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1000. 00000100:00000001:2.0:1773941538.016466:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941538.016473:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241168256/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941554 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941538.016483:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941538.016488:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d63800 x1860112241168256/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941554 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941538.016497:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941538.016499:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941538.017413:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941538.017425:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941538.017428:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941538.017444:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941538.017451:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941538.017457:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941538.017465:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941538.017469:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941538.017475:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941538.017479:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321608f80 00000400:00000200:2.0:1773941538.017485:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941538.017493:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x13bd [1] + 224 00000800:00000001:2.0:1773941538.017501:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941538.017511:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941538.017513:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941538.017517:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941538.017521:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941538.017525:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241168256/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941554 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941538.017535:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d63800 x1860112241168256/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941554 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941538.017558:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941538.017566:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941538.017569:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941538.017593:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941538.017601:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941538.017603:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941538.017606:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941538.017610:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941538.017612:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941538.017614:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941538.017620:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241168256/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941554 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941538.017629:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241168256/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941554 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941538.017635:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941538.017636:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941538.017638:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941538.017641:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941538.017643:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941538.017645:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941538.017647:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941538.017652:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=117 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941538.017655:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941538.017658:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941538.017662:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941538.017664:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941538.017667:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941538.017669:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941538.017673:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941538.017677:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941538.017687:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941538.017689:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941538.017691:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941538.017693:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941538.017696:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d63800 x1860112241168256/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941554 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941538.017701:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941538.017721:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941538.017723:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941538.017725:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941538.017728:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941538.017729:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941538.017731:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941538.017732:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941538.017732:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941538.017734:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941538.017735:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941538.017737:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941538.017739:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941538.017741:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941538.017743:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 306 00000400:00000200:1.0:1773941538.017748:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941538.017751:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941538.017754:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941538.017757:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941538.017759:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941538.017761:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941538.017764:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941538.017765:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941538.017767:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941538.017769:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941538.017771:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941538.017772:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941538.017779:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941538.017780:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941538.017782:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941538.017783:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941538.017785:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941538.017788:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941538.017790:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941538.017792:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941538.017795:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941538.017797:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941538.017799:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941538.017800:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941538.017803:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60e00. 00000020:00000040:1.0:1773941538.017805:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941538.017808:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941538.017810:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941538.017811:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941538.017813:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941538.017815:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941538.017819:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0736800. 00000100:00000001:1.0:1773941538.017822:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941538.017826:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60e00 x1860112241168384/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941538.017832:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941538.017835:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941538.017838:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5e4d003] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60e00 x1860112241168384/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941538.017859:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941538.017861:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941538.017862:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941538.017864:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941538.017866:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941538.017873:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941538.017875:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941538.017879:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941538.017880:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941538.017884:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241168256/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941554 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941538.017887:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941538.017894:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941538.017894:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941538.017903:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941538.017903:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941538.017911:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941538.017914:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941538.017916:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60e00 x1860112241168384/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941538.017923:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241168256/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941554 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941538.017956:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1773941538.017960:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241168256:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941538.017967:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941538.017973:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1773941538.017973:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941538.017984:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:0.0:1773941538.017985:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941538.017997:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:0.0:1773941538.017997:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941538.018006:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:0.0:1773941538.018011:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241168384:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000040:1.0:1773941538.018014:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241168256/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941554 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941538.018024:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941538.018032:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941538.018033:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:1.0:1773941538.018036:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941538.018040:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941538.018043:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:0.0:1773941538.018044:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941538.018061:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941538.018064:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000010:0.0:1773941538.018068:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0c00. 00010000:00000001:1.0:1773941538.018070:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941538.018074:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:0.0:1773941538.018074:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941538.018077:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ec00 refcount=4 00000020:00000001:1.0:1773941538.018083:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00000400:00000200:0.0:1773941538.018083:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941538.018089:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00000400:00000010:0.0:1773941538.018091:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00010000:1.0:1773941538.018097:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e66 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941538.018099:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941538.018105:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241168384, portal 25 00000100:00000001:0.0:1773941538.018113:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941538.018118:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00010000:00000001:1.0:1773941538.018128:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:0.0:1773941538.018134:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60e00 x1860112241168384/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941538.018136:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e66 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941538.018168:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:0.0:1773941538.018171:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:1.0:1773941538.018172:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e66 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:0.0:1773941538.018183:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941538.018189:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c2a8. 00010000:00000001:1.0:1773941538.018197:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000200:0.0:1773941538.018199:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241168384, offset 0 00010000:00000001:1.0:1773941538.018201:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941538.018208:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e66 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941538.018215:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941538.018228:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:0.0:1773941538.018228:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941538.018233:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941538.018237:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:0.0:1773941538.018242:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00010000:1.0:1773941538.018243:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e66 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941538.018256:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:1.0:1773941538.018266:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138ec00 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:0.0:1773941538.018266:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 10000000:00000001:1.0:1773941538.018275:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941538.018280:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00010000:1.0:1773941538.018281:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e66 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941538.018288:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941538.018298:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941538.018302:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:0.0:1773941538.018303:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[188, 189] 00000020:00000001:1.0:1773941538.018311:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1773941538.018313:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000020:00000040:1.0:1773941538.018317:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ec00 refcount=5 00000400:00000200:0.0:1773941538.018321:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [295:295:256:4294967295] 192.168.204.143@tcp LPNI seq info [190:190:8:4294967295] 00000020:00000001:1.0:1773941538.018325:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941538.018333:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00000400:00000200:0.0:1773941538.018336:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941538.018347:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941538.018349:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000200:0.0:1773941538.018354:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941538.018357:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e66 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:0.0:1773941538.018360:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 10000000:00000001:1.0:1773941538.018372:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:0.0:1773941538.018372:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 10000000:00010000:1.0:1773941538.018377:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e66 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941538.018382:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000800:00000200:0.0:1773941538.018384:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 10000000:00000040:1.0:1773941538.018391:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000800:00000200:0.0:1773941538.018392:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:1.0:1773941538.018394:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941538.018407:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941538.018410:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941538.018413:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941538.018414:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941538.018415:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:0.0:1773941538.018416:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941538.018418:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941538.018423:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000040:1.0:1773941538.018430:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138ec00 with handle 0xa33a5be14eae1e66 from hash 00000100:00000001:0.0:1773941538.018431:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294978381 00010000:00000001:1.0:1773941538.018441:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941538.018443:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941538.018445:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941538.018446:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:0.0:1773941538.018447:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294978379 00010000:00000001:1.0:1773941538.018449:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941538.018451:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941538.018457:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941538.018457:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294978381 00010000:00000001:1.0:1773941538.018464:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941538.018467:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941538.018469:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941538.018469:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294978379 00010000:00010000:1.0:1773941538.018477:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e66 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941538.018479:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.0:1773941538.018480:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941538.018488:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000100:00000001:0.0:1773941538.018490:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941538.018499:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1773941538.018502:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941538.018515:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941538.018516:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941538.018523:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941538.018524:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:0.0:1773941538.018527:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941538.018533:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:0.0:1773941538.018533:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:2.0:1773941538.018535:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c2a8 00010000:00000001:1.0:1773941538.018541:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000400:00000010:2.0:1773941538.018542:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c2a8. 00000100:00000001:0.0:1773941538.018542:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941538.018546:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941538.018547:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941538.018553:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941538.018554:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941538.018558:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000200:2.0:1773941538.018560:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60e00 x1860112241168384/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941538.018563:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941538.018563:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941538.018574:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e66 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941538.018576:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941538.018586:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60e00 x1860112241168384/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941548 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000040:1.0:1773941538.018587:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec700 count: 0 00000100:00000001:2.0:1773941538.018600:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941538.018602:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1773941538.018605:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941538.018608:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941538.018611:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941538.018616:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241168256/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941554 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941538.018622:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941538.018625:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941538.018627:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0737000. 02000000:00000001:1.0:1773941538.018630:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941538.018632:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941538.018633:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941538.018635:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941538.018637:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941538.018640:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941538.018641:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941538.018643:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:1.0:1773941538.018645:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941538.018647:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941538.018648:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941538.018650:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941538.018652:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941538.018655:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0ac0. 00000020:00000010:1.0:1773941538.018659:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941538.018665:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941538.018668:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941538.018673:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941538.018676:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941538.018680:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941538.018689:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941538.018693:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941538.018697:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941538.018699:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941538.018735:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941538.018739:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941538.018742:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941538.018747:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941538.018750:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941538.018755:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941538.018758:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941538.018763:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941538.018767:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0ac0. 10000000:01000000:1.0:1773941538.018771:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941538.018774:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941538.018776:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941538.018778:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941538.018780:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941538.018782:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941538.018784:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941538.019631:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941538.019639:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941538.019641:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941538.019643:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941538.019648:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941538.019653:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941538.019659:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941538.019662:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941538.019665:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941538.019668:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321609000 00000400:00000200:2.0:1773941538.019672:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941538.019676:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x13c5 [1] + 0 00000800:00000001:2.0:1773941538.019681:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941538.019688:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941538.019691:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941538.019695:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941538.019699:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941538.019736:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60e00 x1860112241168384/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941548 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941538.019746:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60e00 x1860112241168384/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941548 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941538.019755:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941538.019760:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941538.019763:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941538.019787:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941538.019790:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941538.019794:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941538.019796:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941538.019798:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941538.019801:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941538.019802:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941538.019804:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941538.019809:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60e00 x1860112241168384/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941548 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941538.019828:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60e00 x1860112241168384/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941548 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941538.019833:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941538.019834:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941538.019836:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941538.019839:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941538.019841:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941538.019842:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941538.019844:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941538.019849:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=117 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941538.019851:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941538.019853:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941538.019854:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941538.019856:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941538.019858:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941538.019860:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941538.019863:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60e00 x1860112241168384/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941548 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941538.019867:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941538.019869:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941538.019871:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941538.019872:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941538.019878:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941538.019880:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941538.019882:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941538.019884:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941538.019886:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941538.019888:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941538.019889:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941538.019892:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941538 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941538.019894:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941538.019899:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941538.019904:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941538.019907:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941538.019909:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941538.019910:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941538.019913:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941538.019914:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941538.019916:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941538.019919:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941538.019922:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941538.019924:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941538.020182:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941538.020187:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941538.020194:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60e00 x1860112241168384/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941548 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941538.020203:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241168384:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941538.020203:0:12258:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:1.0:1773941538.020211:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941538.020213:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1773941538.020214:0:12258:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000040:1.0:1773941538.020218:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60e00 x1860112241168384/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941548 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941538.020220:0:12258:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941538.020224:0:12258:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941538.020227:0:12258:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:1.0:1773941538.020232:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941538.020232:0:12258:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:1.0:1773941538.020236:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941538.020236:0:12258:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941538.020240:0:12258:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941538.020243:0:12258:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:1.0:1773941538.020245:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0c00. 00000001:00000001:0.0:1773941538.020245:0:12258:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941538.020248:0:12258:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941538.020250:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:00000001:0.0:1773941538.020251:0:12258:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941538.020253:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941538.020254:0:12258:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:1.0:1773941538.020256:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941538.020259:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:0.0:1773941538.020260:0:12258:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:1.0:1773941538.020262:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0736800. 00000100:00000001:0.0:1773941538.020264:0:12258:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:1.0:1773941538.020266:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941538.020267:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941538.020268:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60e00. 00000100:00000001:1.0:1773941538.020270:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941538.020271:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941538.020275:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941538.020277:0:12258:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941538.020280:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941538.020281:0:12258:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941538.020283:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941538.020287:0:12258:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941538.020288:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941538.020291:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941538.020294:0:12258:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941538.020298:0:12258:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941538.020300:0:12258:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941538.020316:0:12258:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941538.020319:0:12258:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941538.020322:0:12258:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941538.020327:0:12258:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941538.020339:0:12258:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941538.020341:0:12258:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941538.020343:0:12258:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941538.020345:0:12258:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941538.020346:0:12258:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941538.023748:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec700. 00010000:00000010:1.1:1773941538.023754:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138ec00. 00000020:00000001:3.0:1773941538.655909:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941538.655917:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941538.655922:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941539.679830:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941539.679840:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941539.679846:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941539.679850:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941539.679852:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941539.679853:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941539.679856:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941539.679858:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941539.679860:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941539.679863:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941539.679865:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941539.679879:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60e00. 00000020:00000040:1.0:1773941539.679884:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941539.679889:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941539.679893:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:3.0:1773941539.679900:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 02000000:00000001:1.0:1773941539.679900:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941539.679906:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941539.679910:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00010000:00010000:3.0:1773941539.679912:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1773941539.679919:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941539.679926:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941539.679929:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941539.679937:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec700. 00010000:00000001:1.0:1773941539.679943:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941539.679963:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138ec00. 00000020:00000001:1.0:1773941539.679968:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941539.679971:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138ec00 with handle 0xa33a5be14eae1e6d to hash 00000020:00000001:1.0:1773941539.679973:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941539.679974:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941539.679977:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941539.679981:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941539.679984:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941539.679988:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e6d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941539.679997:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e6d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941539.680009:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e6d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941539.680014:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941539.680032:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941539.680040:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941539.680043:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941539.680046:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941539.680047:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00000100:00000001:1.0:1773941539.680054:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941539.680059:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941539.680061:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941539.680063:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941539.680074:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60e00 x1860112241168512/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941539.680082:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941539.680084:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941539.680088:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941539.680090:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941539.680101:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941539.680104:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241168512:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941539.680108:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941539.680114:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941539.680116:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941539.680119:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941539.680123:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0736800. 02000000:00000001:1.0:1773941539.680125:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941539.680133:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941539.680137:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941539.680143:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941539.680150:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241168512, portal 25 00000100:00000001:1.0:1773941539.680152:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941539.680153:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00000100:00000040:1.0:1773941539.680157:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60e00 x1860112241168512/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941555 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941539.680163:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941539.680166:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941539.680168:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f12a8. 00000100:00000200:1.0:1773941539.680171:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241168512, offset 0 00000400:00000200:1.0:1773941539.680177:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941539.680185:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941539.680193:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941539.680199:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941539.680202:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941539.680206:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941539.680208:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941539.680211:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[190, 189] 00000400:00000200:1.0:1773941539.680213:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941539.680217:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [296:296:256:4294967295] 192.168.204.153@tcp LPNI seq info [191:191:8:4294967295] 00000400:00000200:1.0:1773941539.680220:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941539.680229:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941539.680253:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941539.680257:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b800. 00000800:00000200:1.0:1773941539.680264:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941539.680272:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941539.680277:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b800 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941539.680290:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941539.680294:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941539.680297:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294980043 00000100:00000001:1.0:1773941539.680304:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294978381 00000100:00000001:1.0:1773941539.680309:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294980043 00000100:00000001:1.0:1773941539.680315:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294978381 00000100:00000001:1.0:1773941539.680319:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941539.680322:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941539.680325:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941539.680327:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941539.680330:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941539.680333:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941539.680337:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941539.680409:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941539.680414:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b800. 00000400:00000200:2.0:1773941539.680420:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941539.680426:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941539.680430:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f12a8 00000400:00000010:2.0:1773941539.680437:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f12a8. 00000100:00000001:2.0:1773941539.680442:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941539.680446:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60e00 x1860112241168512/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941555 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941539.680455:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941539.680458:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d60e00 x1860112241168512/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941555 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941539.680463:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941539.680465:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941539.681649:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941539.681665:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941539.681669:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941539.681672:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941539.681679:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941539.681685:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941539.681692:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941539.681698:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941539.681724:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941539.681730:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321609080 00000400:00000200:2.0:1773941539.681735:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941539.681739:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x13cd [1] + 224 00000800:00000001:2.0:1773941539.681747:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941539.681755:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941539.681757:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941539.681759:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941539.681763:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941539.681767:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60e00 x1860112241168512/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941555 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941539.681776:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d60e00 x1860112241168512/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941555 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941539.681790:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941539.681805:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941539.681809:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941539.681854:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941539.681860:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941539.681862:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941539.681865:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941539.681868:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941539.681870:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941539.681872:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941539.681877:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60e00 x1860112241168512/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941555 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941539.681885:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60e00 x1860112241168512/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941555 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941539.681891:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941539.681892:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941539.681900:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941539.681903:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941539.681905:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941539.681907:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941539.681909:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941539.681915:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=118 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941539.681918:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941539.681920:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941539.681924:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941539.681927:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941539.681930:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941539.681932:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941539.681936:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941539.681939:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941539.681942:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941539.681944:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941539.681945:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941539.681947:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941539.681950:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d60e00 x1860112241168512/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941555 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941539.681955:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941539.681956:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941539.681958:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941539.681960:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941539.681963:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941539.681964:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941539.681965:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941539.681966:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941539.681967:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941539.681967:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941539.681969:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941539.681971:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941539.681973:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941539.681975:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941539.681980:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 308 00000400:00000200:1.0:1773941539.681985:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941539.681988:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941539.681992:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941539.681995:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941539.681997:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941539.681999:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941539.682002:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941539.682004:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941539.682005:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941539.682007:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941539.682009:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941539.682011:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941539.682012:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941539.682014:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941539.682029:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941539.682031:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941539.682035:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941539.682038:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941539.682041:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941539.682044:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941539.682048:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941539.682051:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941539.682053:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941539.682055:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941539.682059:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941539.682061:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941539.682064:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941539.682065:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941539.682067:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941539.682068:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941539.682070:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941539.682078:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0737000. 00000100:00000001:1.0:1773941539.682081:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941539.682087:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d63800 x1860112241168640/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941539.682093:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941539.682096:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941539.682099:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc928f9c] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d63800 x1860112241168640/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941539.682118:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941539.682120:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941539.682121:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941539.682123:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941539.682125:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941539.682126:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941539.682128:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941539.682130:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941539.682131:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941539.682135:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60e00 x1860112241168512/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941555 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941539.682141:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941539.682144:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941539.682145:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941539.682147:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941539.682150:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941539.682150:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941539.682154:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941539.682157:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60e00 x1860112241168512/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941555 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941539.682164:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241168640/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941539.682167:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241168512:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941539.682176:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941539.682180:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941539.682183:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941539.682184:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941539.682187:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941539.682188:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1773941539.682191:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941539.682194:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1773941539.682199:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60e00 x1860112241168512/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941555 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1773941539.682199:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941539.682207:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241168640:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941539.682211:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941539.682213:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:1.0:1773941539.682215:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 02000000:00000001:0.0:1773941539.682217:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941539.682219:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:0.0:1773941539.682220:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941539.682221:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:0.0:1773941539.682224:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941539.682226:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941539.682229:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000010:0.0:1773941539.682230:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c3000. 00000020:00000001:1.0:1773941539.682231:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941539.682234:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ec00 refcount=4 02000000:00000001:0.0:1773941539.682234:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941539.682239:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00000400:00000200:0.0:1773941539.682243:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941539.682244:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00000400:00000010:0.0:1773941539.682247:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00010000:1.0:1773941539.682249:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e6d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941539.682252:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941539.682258:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241168640, portal 25 00010000:00000001:1.0:1773941539.682259:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941539.682262:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00010000:1.0:1773941539.682265:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e6d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941539.682265:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:0.0:1773941539.682273:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241168640/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941539.682274:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941539.682278:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e6d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941539.682289:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941539.682291:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941539.682295:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000040:0.0:1773941539.682296:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941539.682297:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e6d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941539.682300:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c000. 00010000:00000001:1.0:1773941539.682303:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941539.682305:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941539.682307:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941539.682310:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e6d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:0.0:1773941539.682314:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241168640, offset 0 00000400:00000200:0.0:1773941539.682325:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941539.682334:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138ec00 already unused, calling callback (ffffffffc1482c20) 00000400:00000200:0.0:1773941539.682334:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:00000001:1.0:1773941539.682340:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941539.682343:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 10000000:00010000:1.0:1773941539.682345:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e6d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941539.682349:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941539.682356:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00010000:00000001:1.0:1773941539.682358:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941539.682363:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:0.0:1773941539.682363:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1773941539.682366:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941539.682370:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ec00 refcount=5 00000400:00000200:0.0:1773941539.682371:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1773941539.682374:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941539.682377:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941539.682381:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941539.682385:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e6d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941539.682391:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[190, 191] 10000000:00000001:1.0:1773941539.682401:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941539.682404:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 10000000:00010000:1.0:1773941539.682405:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e6d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941539.682413:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [297:297:256:4294967295] 192.168.204.143@tcp LPNI seq info [192:192:8:4294967295] 10000000:01000000:1.0:1773941539.682415:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941539.682420:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941539.682423:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941539.682427:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:0.0:1773941539.682427:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941539.682431:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941539.682434:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941539.682437:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941539.682441:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:0.0:1773941539.682442:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:1.0:1773941539.682444:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941539.682448:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138ec00 with handle 0xa33a5be14eae1e6d from hash 00000800:00000200:0.0:1773941539.682451:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:1.0:1773941539.682453:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941539.682456:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941539.682457:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:0.0:1773941539.682458:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00010000:00000001:1.0:1773941539.682464:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000800:00000200:0.0:1773941539.682465:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941539.682467:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941539.682471:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941539.682473:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:0.0:1773941539.682475:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:1.0:1773941539.682476:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941539.682479:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941539.682481:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:1.0:1773941539.682482:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941539.682487:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e6d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941539.682492:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941539.682492:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941539.682493:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941539.682495:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:0.0:1773941539.682496:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941539.682497:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941539.682500:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941539.682501:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941539.682503:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e6d lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941539.682508:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec700 count: 0 00000400:00000200:0.0:1773941539.682509:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941539.682513:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294980045 00000100:00000001:0.0:1773941539.682518:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294980043 00000100:00000001:0.0:1773941539.682523:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294980045 00000100:00000001:0.0:1773941539.682526:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294980043 00010000:00000001:1.0:1773941539.682530:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941539.682530:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941539.682535:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941539.682535:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941539.682540:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941539.682542:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941539.682544:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941539.682551:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60e00 x1860112241168512/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941555 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941539.682552:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941539.682554:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000200:2.0:1773941539.682560:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1773941539.682561:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000800:00000010:2.0:1773941539.682567:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 02000000:00000001:1.0:1773941539.682568:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1773941539.682568:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941539.682573:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1773941539.682574:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:1.0:1773941539.682574:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0736800. 00000100:00000001:0.0:1773941539.682575:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941539.682582:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941539.682583:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:2.0:1773941539.682586:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000020:00000001:1.0:1773941539.682588:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:2.0:1773941539.682590:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c000 00000020:00000040:1.0:1773941539.682592:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000400:00000010:2.0:1773941539.682594:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c000. 00000020:00000001:1.0:1773941539.682598:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941539.682599:0:2347:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:1.0:1773941539.682603:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 00000100:00000200:2.0:1773941539.682607:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241168640/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941539.682608:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941539.682612:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941539.682617:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941539.682622:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241168640/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941549 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941539.682622:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60e00. 00000100:00000001:1.0:1773941539.682628:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941539.682631:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941539.682633:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941539.682634:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941539.682636:0:2347:0:(events.c:68:request_out_callback()) Process leaving 10000000:01000000:1.0:1773941539.682638:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941539.682642:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941539.682646:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0ac0. 00000020:00000010:1.0:1773941539.682649:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941539.682655:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941539.682659:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941539.682664:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941539.682668:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941539.682672:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941539.682676:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941539.682680:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6e00. 10000000:00000040:1.0:1773941539.682684:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941539.682686:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941539.682690:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6a00. 00000040:00000010:1.0:1773941539.682694:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b7800. 00000020:00000010:1.0:1773941539.682696:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941539.682700:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941539.682732:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941539.682737:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b6800. 20000000:00000010:1.0:1773941539.682739:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6e00. 00000020:00000010:1.0:1773941539.682742:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941539.682744:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0ac0. 10000000:01000000:1.0:1773941539.682747:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941539.682750:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941539.682752:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941539.682754:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941539.682755:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941539.682756:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941539.682758:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941539.684120:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941539.684126:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941539.684128:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941539.684129:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941539.684133:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941539.684136:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941539.684141:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941539.684143:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941539.684145:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941539.684148:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321609100 00000400:00000200:2.0:1773941539.684150:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941539.684154:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x13d5 [1] + 0 00000800:00000001:2.0:1773941539.684158:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941539.684162:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941539.684164:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941539.684167:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941539.684169:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941539.684179:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241168640/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941549 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941539.684186:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d63800 x1860112241168640/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941549 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941539.684200:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941539.684204:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941539.684206:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941539.684230:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941539.684232:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941539.684235:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941539.684236:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941539.684238:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941539.684241:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941539.684243:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941539.684245:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941539.684250:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241168640/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941549 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941539.684257:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241168640/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941549 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941539.684263:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941539.684264:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941539.684265:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941539.684267:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941539.684268:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941539.684269:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941539.684270:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941539.684273:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=118 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941539.684275:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941539.684277:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941539.684279:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941539.684281:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941539.684283:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941539.684285:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941539.684288:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241168640/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941549 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941539.684294:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941539.684295:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941539.684296:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941539.684314:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941539.684318:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941539.684320:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941539.684321:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941539.684323:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941539.684325:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941539.684326:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941539.684328:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941539.684332:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941539 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941539.684334:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941539.684339:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941539.684341:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941539.684342:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941539.684344:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941539.684345:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941539.684348:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941539.684350:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941539.684352:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941539.684355:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941539.684357:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941539.684359:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941539.684618:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941539.684627:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941539.684633:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241168640/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941549 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941539.684640:0:12259:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:1.0:1773941539.684641:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241168640:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941539.684647:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941539.684649:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941539.684652:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241168640/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941549 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941539.684655:0:12259:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00000001:1.0:1773941539.684665:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:0.0:1773941539.684665:0:12259:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:1.0:1773941539.684667:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941539.684670:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c3000. 00000100:00000001:0.0:1773941539.684672:0:12259:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:1.0:1773941539.684674:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:01000000:0.0:1773941539.684676:0:12259:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000001:1.0:1773941539.684677:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941539.684679:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941539.684682:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:0.0:1773941539.684683:0:12259:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:1.0:1773941539.684684:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0737000. 02000000:00000001:1.0:1773941539.684688:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:0.0:1773941539.684690:0:12259:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:1.0:1773941539.684691:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941539.684694:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:1.0:1773941539.684697:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941539.684699:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:0.0:1773941539.684699:0:12259:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941539.684729:0:12259:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:1.0:1773941539.684731:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:0.0:1773941539.684732:0:12259:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:1.0:1773941539.684737:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000001:00000001:0.0:1773941539.684737:0:12259:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941539.684739:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941539.684742:0:12259:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941539.684743:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941539.684746:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941539.684748:0:12259:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941539.684763:0:12259:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941539.684766:0:12259:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941539.684769:0:12259:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941539.684772:0:12259:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941539.684775:0:12259:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941539.684777:0:12259:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941539.684779:0:12259:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941539.684781:0:12259:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941539.684798:0:12259:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941539.684800:0:12259:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941539.684803:0:12259:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941539.684808:0:12259:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941539.684813:0:12259:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941539.684816:0:12259:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941539.684818:0:12259:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941539.684821:0:12259:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941539.684823:0:12259:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941539.687753:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec700. 00010000:00000010:1.1:1773941539.687759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138ec00. 00000100:00000040:2.0:1773941539.743834:0:37:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (315) 00000020:00000001:3.0:1773941540.703950:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941540.703978:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941540.703983:0:12142:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941541.087945:0:12144:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941541.087960:0:12144:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941541.087968:0:12144:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941541.087971:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941541.087973:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941541.087975:0:12144:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941541.087979:0:12144:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941541.087982:0:12144:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941541.087985:0:12144:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff8b5b982fb370 from 1 10000000:00000001:1.0:1773941541.088049:0:12144:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941541.088051:0:12144:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941541.088062:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d63800. 00000020:00000040:1.0:1773941541.088069:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941541.088076:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941541.088097:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941541.088106:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941541.088116:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b850b7000. 00000100:00000001:1.0:1773941541.088126:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941541.088133:0:12144:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941541.088137:0:12144:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941541.088147:0:12144:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5b890ec700. 00010000:00000001:1.0:1773941541.088155:0:12144:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941541.088159:0:12144:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5b8138ec00. 00000020:00000001:1.0:1773941541.088164:0:12144:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941541.088168:0:12144:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5b8138ec00 with handle 0xa33a5be14eae1e74 to hash 00000020:00000001:1.0:1773941541.088170:0:12144:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941541.088172:0:12144:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941541.088175:0:12144:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941541.088181:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941541.088186:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941541.088190:0:12144:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e74 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941541.088202:0:12144:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e74 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941541.088217:0:12144:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e74 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941541.088224:0:12144:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941541.088226:0:12144:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941541.088234:0:12144:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941541.088237:0:12144:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824258982272 : -128249450569344 : ffff8b5b990e6180) 00000100:00000001:1.0:1773941541.088246:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941541.088248:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000001:1.0:1773941541.088255:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941541.088261:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.088264:0:12144:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941541.088267:0:12144:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941541.088281:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d63800 x1860112241168768/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.088291:0:12144:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941541.088294:0:12144:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941541.088309:0:12144:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941541.088313:0:12144:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941541.088316:0:12144:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941541.088320:0:12144:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241168768:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941541.088325:0:12144:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941541.088334:0:12144:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941541.088338:0:12144:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941541.088342:0:12144:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941541.088366:0:12144:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0737000. 02000000:00000001:1.0:1773941541.088369:0:12144:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941541.088381:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:1.0:1773941541.088388:0:12144:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941541.088399:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00000100:00000200:1.0:1773941541.088411:0:12144:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241168768, portal 25 00000100:00000001:1.0:1773941541.088414:0:12144:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941541.088416:0:12144:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121083904 : -128249588467712 : ffff8b5b90d63800) 00000100:00000040:1.0:1773941541.088423:0:12144:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d63800 x1860112241168768/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941557 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.088431:0:12144:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941541.088434:0:12144:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:1.0:1773941541.088438:0:12144:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f1330. 00000100:00000200:1.0:1773941541.088442:0:12144:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241168768, offset 0 00000400:00000200:1.0:1773941541.088449:0:12144:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:1.0:1773941541.088467:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941541.088478:0:12144:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:1.0:1773941541.088484:0:12144:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941541.088489:0:12144:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:1.0:1773941541.088494:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941541.088497:0:12144:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941541.088500:0:12144:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[192, 191] 00000400:00000200:1.0:1773941541.088504:0:12144:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:1.0:1773941541.088508:0:12144:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [298:298:256:4294967295] 192.168.204.153@tcp LPNI seq info [193:193:8:4294967295] 00000400:00000200:1.0:1773941541.088512:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:1.0:1773941541.088524:0:12144:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:1.0:1773941541.088530:0:12144:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:1.0:1773941541.088535:0:12144:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024b100. 00000800:00000200:1.0:1773941541.088544:0:12144:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:1.0:1773941541.088555:0:12144:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:1.0:1773941541.088562:0:12144:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024b100 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941541.088589:0:12144:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941541.088595:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941541.088599:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294981451 00000100:00000001:1.0:1773941541.088610:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294980045 00000100:00000001:1.0:1773941541.088618:0:12144:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294981451 00000100:00000001:1.0:1773941541.088623:0:12144:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294980045 00000100:00000001:1.0:1773941541.088629:0:12144:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941541.088650:0:12144:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.088655:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941541.088658:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941541.088665:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 11 seconds 00000100:00000001:1.0:1773941541.088671:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941541.088676:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941541.088971:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941541.088985:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024b100. 00000400:00000200:2.0:1773941541.089016:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941541.089039:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941541.089048:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f1330 00000400:00000010:2.0:1773941541.089051:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f1330. 00000100:00000001:2.0:1773941541.089058:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941541.089069:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d63800 x1860112241168768/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941557 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941541.089086:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941541.089101:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b90d63800 x1860112241168768/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941557 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941541.089112:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941541.089114:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941541.090549:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.090569:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941541.090574:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.090580:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941541.090592:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941541.090604:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941541.090629:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941541.090638:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941541.090648:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.090657:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321609180 00000400:00000200:2.0:1773941541.090667:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.090690:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x13dd [1] + 224 00000800:00000001:2.0:1773941541.090701:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.090744:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941541.090749:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941541.090754:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941541.090760:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941541.090767:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d63800 x1860112241168768/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941557 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941541.090782:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b90d63800 x1860112241168768/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941557 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941541.090817:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941541.090826:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.090830:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941541.090896:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941541.090909:0:12144:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941541.090926:0:12144:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941541.090930:0:12144:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941541.090940:0:12144:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941541.090944:0:12144:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941541.090949:0:12144:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941541.090957:0:12144:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d63800 x1860112241168768/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941557 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941541.090969:0:12144:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d63800 x1860112241168768/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941557 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.090977:0:12144:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941541.090979:0:12144:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941541.090982:0:12144:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941541.090986:0:12144:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941541.091005:0:12144:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941541.091009:0:12144:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941541.091013:0:12144:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941541.091022:0:12144:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=120 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941541.091027:0:12144:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941541.091030:0:12144:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941541.091036:0:12144:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:1.0:1773941541.091039:0:12144:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941541.091044:0:12144:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941541.091047:0:12144:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941541.091053:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941541.091057:0:12144:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941541.091062:0:12144:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941541.091064:0:12144:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941541.091066:0:12144:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941541.091069:0:12144:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941541.091074:0:12144:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b90d63800 x1860112241168768/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941557 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.091081:0:12144:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941541.091089:0:12144:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941541.091092:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941541.091095:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941541.091099:0:12144:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941541.091101:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.091102:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941541.091104:0:12144:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941541.091105:0:12144:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941541.091107:0:12144:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.091108:0:12144:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941541.091111:0:12144:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941541.091113:0:12144:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941541.091117:0:12144:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941541.091120:0:12144:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 310 00000400:00000200:1.0:1773941541.091126:0:12144:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:1.0:1773941541.091131:0:12144:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941541.091136:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941541.091139:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941541.091142:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941541.091144:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:1.0:1773941541.091149:0:12144:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941541.091151:0:12144:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941541.091154:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941541.091156:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941541.091159:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941541.091161:0:12144:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:1.0:1773941541.091164:0:12144:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941541.091165:0:12144:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941541.091167:0:12144:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:1.0:1773941541.091169:0:12144:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:1.0:1773941541.091172:0:12144:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:1.0:1773941541.091176:0:12144:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:1.0:1773941541.091178:0:12144:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941541.091185:0:12144:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941541.091189:0:12144:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941541.091191:0:12144:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.091194:0:12144:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941541.091197:0:12144:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941541.091201:0:12144:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90d60e00. 00000020:00000040:1.0:1773941541.091204:0:12144:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:1.0:1773941541.091208:0:12144:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941541.091210:0:12144:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.091212:0:12144:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941541.091214:0:12144:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941541.091217:0:12144:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941541.091221:0:12144:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5ba0736800. 00000100:00000001:1.0:1773941541.091226:0:12144:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941541.091233:0:12144:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90d60e00 x1860112241168896/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941541.091241:0:12144:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941541.091244:0:12144:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941541.091250:0:12144:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e5e4d003] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90d60e00 x1860112241168896/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.091275:0:12144:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.091278:0:12144:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941541.091279:0:12144:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941541.091282:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941541.091285:0:12144:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941541.091287:0:12144:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941541.091289:0:12144:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5b990e6180 going to sleep for 1 seconds 00000100:00000001:1.0:1773941541.091291:0:12144:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941541.091293:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941541.091298:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d63800 x1860112241168768/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941557 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.091307:0:12144:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941541.091310:0:12144:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941541.091319:0:12144:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941541.091319:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941541.091326:0:12144:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941541.091328:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941541.091333:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941541.091335:0:12144:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d63800 x1860112241168768/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941557 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941541.091361:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90d60e00 x1860112241168896/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941541.091377:0:12144:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d63800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:1080e6b5-6a6c-4762-b0ec-c785990fec17:12144:1860112241168768:192.168.204.143@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941541.091384:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941541.091389:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941541.091393:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941541.091396:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941541.091398:0:12144:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1773941541.091401:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.091404:0:12144:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:0.0:1773941541.091411:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241168896:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941541.091412:0:12144:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941541.091417:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941541.091423:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1773941541.091424:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d63800 x1860112241168768/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941557 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1773941541.091430:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941541.091434:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.091436:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941541.091438:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:1.0:1773941541.091440:0:12144:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5b990e6180. 00000100:00000001:1.0:1773941541.091445:0:12144:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941541.091447:0:12144:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000010:0.0:1773941541.091447:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5b850c0000. 02000000:00000001:0.0:1773941541.091453:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941541.091455:0:12144:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941541.091459:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941541.091461:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941541.091464:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ec00 refcount=4 00000020:00000001:1.0:1773941541.091468:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00000400:00000200:0.0:1773941541.091474:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:1.0:1773941541.091475:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00000400:00000010:0.0:1773941541.091480:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00010000:00010000:1.0:1773941541.091482:0:12144:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e74 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941541.091489:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00010000:00000001:1.0:1773941541.091492:0:12144:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000200:0.0:1773941541.091496:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241168896, portal 25 00010000:00010000:1.0:1773941541.091498:0:12144:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e74 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941541.091501:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941541.091505:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824121073152 : -128249588478464 : ffff8b5b90d60e00) 00010000:00000001:1.0:1773941541.091514:0:12144:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000040:0.0:1773941541.091519:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90d60e00 x1860112241168896/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941541.091520:0:12144:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e74 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941541.091527:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941541.091536:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941541.091542:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c5d8. 00010000:00000001:1.0:1773941541.091545:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000200:0.0:1773941541.091547:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241168896, offset 0 00010000:00000001:1.0:1773941541.091549:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:0.0:1773941541.091554:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00010000:1.0:1773941541.091556:0:12144:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e74 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941541.091564:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941541.091573:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:1.0:1773941541.091578:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:0.0:1773941541.091579:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941541.091586:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941541.091587:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941541.091593:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941541.091596:0:12144:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:0.0:1773941541.091597:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941541.091603:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[192, 193] 00010000:00010000:1.0:1773941541.091604:0:12144:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e74 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941541.091609:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00000400:00000200:0.0:1773941541.091618:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [299:299:256:4294967295] 192.168.204.143@tcp LPNI seq info [194:194:8:4294967295] 00000400:00000200:0.0:1773941541.091645:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:1.0:1773941541.091646:0:12144:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5b8138ec00 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:1.0:1773941541.091656:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:0.0:1773941541.091657:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 10000000:00010000:1.0:1773941541.091662:0:12144:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e74 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941541.091667:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00000800:00000010:0.0:1773941541.091676:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00010000:00000001:1.0:1773941541.091687:0:12144:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000800:00000200:0.0:1773941541.091688:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:1.0:1773941541.091694:0:12144:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941541.091700:0:12144:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:0.0:1773941541.091706:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00000020:00000040:1.0:1773941541.091707:0:12144:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5b8138ec00 refcount=5 00000800:00000200:0.0:1773941541.091713:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 616 niov 1 nkiov 1 00000020:00000001:1.0:1773941541.091715:0:12144:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941541.091758:0:12144:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615823859117056 : -128249850434560 : ffff8b5b8138ec00) 00010000:00000001:1.0:1773941541.091762:0:12144:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941541.091766:0:12144:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e74 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:1.0:1773941541.091775:0:12144:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941541.091778:0:12144:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e74 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941541.091779:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941541.091792:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 10000000:01000000:1.0:1773941541.091796:0:12144:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:0.0:1773941541.091804:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294981455 10000000:00000040:1.0:1773941541.091812:0:12144:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941541.091817:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.091817:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294981451 00010000:00010000:1.0:1773941541.091830:0:12144:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:0.0:1773941541.091830:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294981455 00010000:00000001:1.0:1773941541.091838:0:12144:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:0.0:1773941541.091840:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294981451 00000100:00000001:0.0:1773941541.091847:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941541.091848:0:12144:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:0.0:1773941541.091850:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.091855:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.091858:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941541.091866:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941541.091868:0:12144:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:0.0:1773941541.091869:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1773941541.091876:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1773941541.091877:0:12144:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:0.0:1773941541.091879:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941541.091885:0:12144:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.091886:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.091896:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941541.091898:0:12144:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5b8138ec00 with handle 0xa33a5be14eae1e74 from hash 00010000:00000001:1.0:1773941541.091906:0:12144:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941541.091917:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941541.091921:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:2.0:1773941541.091925:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941541.091926:0:12144:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941541.091937:0:12144:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000800:00000010:2.0:1773941541.091939:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00010000:00000001:1.0:1773941541.091945:0:12144:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:2.0:1773941541.091951:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:1.0:1773941541.091952:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941541.091960:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941541.091964:0:12144:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941541.091974:0:12144:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941541.091975:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00010000:00010000:1.0:1773941541.092006:0:12144:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e74 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941541.092012:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c5d8 00000400:00000010:2.0:1773941541.092019:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c5d8. 00000100:00000001:2.0:1773941541.092032:0:2347:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941541.092035:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941541.092039:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941541.092044:0:12144:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000200:2.0:1773941541.092047:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90d60e00 x1860112241168896/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941541.092048:0:12144:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941541.092054:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941541.092059:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941541.092065:0:12144:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941541.092072:0:12144:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5b8138ec00/0xa33a5be14eae1e74 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12144 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941541.092092:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000040:1.0:1773941541.092099:0:12144:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5b890ec700 count: 0 00000100:00000040:2.0:1773941541.092103:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90d60e00 x1860112241168896/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941541.092116:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941541.092120:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1773941541.092127:0:12144:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941541.092132:0:12144:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941541.092136:0:12144:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941541.092154:0:12144:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d63800 x1860112241168768/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941557 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.092163:0:12144:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941541.092166:0:12144:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941541.092170:0:12144:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0737000. 02000000:00000001:1.0:1773941541.092174:0:12144:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941541.092176:0:12144:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941541.092178:0:12144:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941541.092180:0:12144:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941541.092184:0:12144:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b850b7000. 02000000:00000001:1.0:1773941541.092188:0:12144:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941541.092190:0:12144:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941541.092192:0:12144:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d63800. 00000100:00000001:1.0:1773941541.092195:0:12144:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941541.092197:0:12144:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941541.092199:0:12144:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941541.092202:0:12144:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941541.092205:0:12144:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941541.092210:0:12144:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b850b0ac0. 00000020:00000010:1.0:1773941541.092214:0:12144:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5b850b7000. 00000020:00000010:1.0:1773941541.092221:0:12144:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941541.092226:0:12144:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941541.092231:0:12144:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941541.092237:0:12144:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941541.092242:0:12144:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941541.092248:0:12144:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941541.092253:0:12144:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5b850b6a00. 10000000:00000040:1.0:1773941541.092258:0:12144:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 10000000:00000001:1.0:1773941541.092261:0:12144:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941541.092266:0:12144:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b850b6e00. 00000040:00000010:1.0:1773941541.092270:0:12144:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b850b6800. 00000020:00000010:1.0:1773941541.092274:0:12144:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b885e2000. 00000020:00000010:1.0:1773941541.092279:0:12144:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b850b7a00. 80000000:00000010:1.0:1773941541.092282:0:12144:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f96c0. 00040000:00000010:1.0:1773941541.092295:0:12144:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b850b7800. 20000000:00000010:1.0:1773941541.092298:0:12144:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5b850b6a00. 00000020:00000010:1.0:1773941541.092302:0:12144:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b850b7000. 10000000:00000010:1.0:1773941541.092305:0:12144:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b850b0ac0. 10000000:01000000:1.0:1773941541.092308:0:12144:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941541.092312:0:12144:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941541.092314:0:12144:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941541.092316:0:12144:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941541.092318:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941541.092319:0:12144:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 2 10000000:00000001:1.0:1773941541.092338:0:12144:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:2.0:1773941541.094269:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.094285:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941541.094289:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.094292:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941541.094300:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.094320:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941541.094327:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.094333:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.094338:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.094342:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321609200 00000400:00000200:2.0:1773941541.094348:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.094354:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x13e5 [1] + 0 00000800:00000001:2.0:1773941541.094371:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.094381:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941541.094385:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941541.094390:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941541.094395:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941541.094403:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90d60e00 x1860112241168896/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941541.094418:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90d60e00 x1860112241168896/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941541.094447:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941541.094456:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.094459:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941541.094498:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941541.094502:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941541.094507:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941541.094509:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941541.094512:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941541.094517:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941541.094519:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941541.094523:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941541.094530:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90d60e00 x1860112241168896/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941541.094541:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90d60e00 x1860112241168896/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941541.094548:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941541.094549:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941541.094551:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941541.094554:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941541.094555:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941541.094556:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941541.094558:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941541.094562:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=120 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941541.094565:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941541.094576:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941541.094578:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941541.094580:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941541.094582:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.094585:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941541.094590:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90d60e00 x1860112241168896/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941541.094596:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941541.094599:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941541.094600:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941541.094603:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1773941541.094608:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:0.0:1773941541.094610:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941541.094611:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:0.0:1773941541.094612:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941541.094614:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:0.0:1773941541.094616:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:0.0:1773941541.094619:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:0.0:1773941541.094623:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941541 v=0 (0 0 0 0) 00000100:00001000:0.0:1773941541.094626:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:0.0:1773941541.094634:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:0.0:1773941541.094636:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941541.094637:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:0.0:1773941541.094639:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.094641:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941541.094644:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:0.0:1773941541.094647:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:0.0:1773941541.094649:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:0.0:1773941541.094653:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:0.0:1773941541.094656:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:0.0:1773941541.094659:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000100:00000001:1.0:1773941541.095049:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.095055:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941541.095062:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90d60e00 x1860112241168896/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941551 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941541.095090:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90d60e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241168896:192.168.204.143@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941541.095094:0:12260:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:1.0:1773941541.095097:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941541.095101:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941541.095104:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90d60e00 x1860112241168896/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941551 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.095115:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00080000:0.0:1773941541.095118:0:12260:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 02000000:00000001:1.0:1773941541.095119:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941541.095125:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5b850c0000. 00000100:00100000:0.0:1773941541.095128:0:12260:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:1.0:1773941541.095131:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1773941541.095132:0:12260:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000001:1.0:1773941541.095135:0:2351:0:(genops.c:1236:class_import_put()) Process entered 10000000:01000000:0.0:1773941541.095137:0:12260:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000040:1.0:1773941541.095140:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000001:00000001:0.0:1773941541.095143:0:12260:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:1.0:1773941541.095145:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:0.0:1773941541.095148:0:12260:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:1.0:1773941541.095150:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5ba0736800. 00000001:00000001:0.0:1773941541.095155:0:12260:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941541.095156:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941541.095161:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:0.0:1773941541.095162:0:12260:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000010:1.0:1773941541.095164:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90d60e00. 00000001:00000001:0.0:1773941541.095168:0:12260:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:1.0:1773941541.095170:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:0.0:1773941541.095172:0:12260:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.095174:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1773941541.095177:0:12260:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.095179:0:12260:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941541.095184:0:12260:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941541.095192:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941541.095192:0:12260:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941541.095198:0:12260:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941541.095201:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941541.095203:0:12260:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.095205:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941541.095208:0:12260:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941541.095211:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941541.095214:0:12260:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:1.0:1773941541.095215:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:0.0:1773941541.095219:0:12260:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941541.095223:0:12260:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:0.0:1773941541.095241:0:12260:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941541.095242:0:12260:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.095244:0:12260:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941541.095247:0:12260:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:0.0:1773941541.095251:0:12260:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941541.095254:0:12260:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941541.095255:0:12260:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:0.0:1773941541.095258:0:12260:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941541.095260:0:12260:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941541.097758:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5b890ec700. 00010000:00000010:1.1:1773941541.097769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5b8138ec00. 00000020:00000001:3.0:1773941541.727848:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941541.727860:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941541.727868:0:12141:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:2.0:1773941541.983848:0:12130:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941541.983861:0:12130:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-client-ffff8b5ba0b09800 from 1 10000000:00000001:2.0:1773941541.983868:0:12130:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941541.983870:0:12130:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00000100:00000010:2.0:1773941541.983900:0:12130:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b9021dc00. 00000020:00000040:2.0:1773941541.983910:0:12130:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000100:00000001:2.0:1773941541.983917:0:12130:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941541.983922:0:12130:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941541.983930:0:12130:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941541.983942:0:12130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff8b5b90e7e400. 00000100:00000001:2.0:1773941541.983952:0:12130:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941541.983959:0:12130:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941541.983963:0:12130:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941541.984006:0:12130:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff8b5ba0d64500. 00010000:00000001:2.0:1773941541.984037:0:12130:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941541.984042:0:12130:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff8b5bc2150600. 00000020:00000001:2.0:1773941541.984048:0:12130:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941541.984052:0:12130:0:(lustre_handles.c:77:class_handle_hash()) added object ffff8b5bc2150600 with handle 0xa33a5be14eae1e7b to hash 00000020:00000001:2.0:1773941541.984055:0:12130:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941541.984056:0:12130:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446615824947283456 : -128248762268160 : ffff8b5bc2150600) 00010000:00000001:2.0:1773941541.984060:0:12130:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446615824947283456 : -128248762268160 : ffff8b5bc2150600) 00010000:00000001:2.0:1773941541.984066:0:12130:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941541.984070:0:12130:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941541.984076:0:12130:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1e7b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941541.984090:0:12130:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1e7b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941541.984106:0:12130:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1e7b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941541.984112:0:12130:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941541.984114:0:12130:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941541.984122:0:12130:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b5ba0b543c0. 00000100:00000001:2.0:1773941541.984124:0:12130:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446615824387359680 : -128249322191936 : ffff8b5ba0b543c0) 00000100:00000001:2.0:1773941541.984127:0:12130:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941541.984128:0:12130:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109263872 : -128249600287744 : ffff8b5b9021dc00) 00000100:00000001:2.0:1773941541.984134:0:12130:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941541.984138:0:12130:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941541.984141:0:12130:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941541.984142:0:12130:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941541.984154:0:12130:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b9021dc00 x1860112241169024/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941541.984162:0:12130:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941541.984165:0:12130:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941541.984169:0:12130:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941541.984176:0:12130:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941541.984178:0:12130:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941541.984182:0:12130:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b9021dc00 pname:cluuid:pid:xid:nid:opc:job mount.lustre:1080e6b5-6a6c-4762-b0ec-c785990fec17:12130:1860112241169024:192.168.204.143@tcp:101:mount.lustre.0 00000100:00000001:2.0:1773941541.984186:0:12130:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941541.984193:0:12130:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941541.984195:0:12130:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941541.984198:0:12130:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941541.984202:0:12130:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0ce7000. 02000000:00000001:2.0:1773941541.984204:0:12130:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941541.984213:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:2.0:1773941541.984217:0:12130:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0af9738. 00000400:00000010:2.0:1773941541.984225:0:12130:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f346e8. 00000100:00000200:2.0:1773941541.984245:0:12130:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241169024, portal 25 00000100:00000001:2.0:1773941541.984247:0:12130:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941541.984248:0:12130:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109263872 : -128249600287744 : ffff8b5b9021dc00) 00000100:00000040:2.0:1773941541.984253:0:12130:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b9021dc00 x1860112241169024/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941557 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941541.984259:0:12130:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941541.984262:0:12130:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:2.0:1773941541.984264:0:12130:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b83f34e58. 00000100:00000200:2.0:1773941541.984267:0:12130:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112241169024, offset 0 00000400:00000200:2.0:1773941541.984273:0:12130:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:2.0:1773941541.984283:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.984292:0:12130:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:2.0:1773941541.984297:0:12130:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941541.984302:0:12130:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:2.0:1773941541.984306:0:12130:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941541.984308:0:12130:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941541.984311:0:12130:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[194, 193] 00000400:00000200:2.0:1773941541.984314:0:12130:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:2.0:1773941541.984317:0:12130:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [300:300:256:4294967295] 192.168.204.153@tcp LPNI seq info [195:195:8:4294967295] 00000400:00000200:2.0:1773941541.984321:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941541.984335:0:12130:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:2.0:1773941541.984341:0:12130:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:2.0:1773941541.984347:0:12130:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b98bb0800. 00000800:00000200:2.0:1773941541.984355:0:12130:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:2.0:1773941541.984364:0:12130:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:2.0:1773941541.984369:0:12130:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b98bb0800 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941541.984391:0:12130:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941541.984395:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:2.0:1773941541.984398:0:12130:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294982347 00000100:00000001:2.0:1773941541.984406:0:12130:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294981455 00000100:00000001:2.0:1773941541.984412:0:12130:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294982347 00000100:00000001:2.0:1773941541.984416:0:12130:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294981455 00000100:00000001:2.0:1773941541.984419:0:12130:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941541.984422:0:12130:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941541.984424:0:12130:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941541.984426:0:12130:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941541.984429:0:12130:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b543c0 going to sleep for 11 seconds 00000100:00000001:2.0:1773941541.984432:0:12130:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941541.984435:0:12130:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941541.984549:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1773941541.984555:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b98bb0800. 00000400:00000200:3.0:1773941541.984562:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1773941541.984568:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:3.0:1773941541.984574:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f34e58 00000400:00000010:3.0:1773941541.984576:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f34e58. 00000100:00000001:3.0:1773941541.984580:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1773941541.984585:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b9021dc00 x1860112241169024/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941557 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941541.984595:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941541.984598:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff8b5b9021dc00 x1860112241169024/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941557 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941541.984604:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.984606:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941541.985505:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.985533:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941541.985537:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.985540:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941541.985547:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941541.985554:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941541.985562:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941541.985567:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941541.985573:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.985578:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321609280 00000400:00000200:2.0:1773941541.985584:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.985591:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x13ed [1] + 224 00000800:00000001:2.0:1773941541.985615:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.985626:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941541.985628:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941541.985633:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941541.985639:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941541.985645:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b9021dc00 x1860112241169024/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941557 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941541.985658:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff8b5b9021dc00 x1860112241169024/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941557 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941541.985679:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941541.985688:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.985693:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941541.985724:0:12130:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941541.985732:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.985734:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.985736:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0af9738. 00000400:00000200:3.0:1773941541.985740:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b83f346e8 00000400:00000010:3.0:1773941541.985741:0:12130:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b83f346e8. 00000100:00000001:3.0:1773941541.985743:0:12130:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941541.985748:0:12130:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b9021dc00 x1860112241169024/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941557 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941541.985756:0:12130:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b9021dc00 x1860112241169024/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941557 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941541.985762:0:12130:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941541.985763:0:12130:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941541.985765:0:12130:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941541.985768:0:12130:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941541.985777:0:12130:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941541.985779:0:12130:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941541.985783:0:12130:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941541.985789:0:12130:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=120 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941541.985792:0:12130:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941541.985795:0:12130:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941541.985799:0:12130:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.143@tcp: operation ldlm_enqueue to node 192.168.204.143@tcp failed: rc = -107 00000100:00000001:3.0:1773941541.985802:0:12130:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941541.985805:0:12130:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941541.985818:0:12130:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.143@tcp of MGS@192.168.204.143@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941541.985823:0:12130:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941541.985827:0:12130:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.143@tcp: Connection to MGS (at 192.168.204.143@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941541.985830:0:12130:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.143@tcp for MGC192.168.204.143@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941541.985832:0:12130:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941541.985833:0:12130:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941541.985835:0:12130:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941541.985838:0:12130:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b5b9021dc00 x1860112241169024/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941557 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941541.985843:0:12130:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941541.985844:0:12130:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941541.985846:0:12130:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941541.985849:0:12130:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941541.985851:0:12130:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941541.985852:0:12130:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.985854:0:12130:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941541.985854:0:12130:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941541.985855:0:12130:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941541.985856:0:12130:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.985857:0:12130:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941541.985859:0:12130:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941541.985861:0:12130:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941541.985864:0:12130:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941541.985866:0:12130:0:(import.c:509:import_select_connection()) MGC192.168.204.143@tcp: connect to NID 192.168.204.143@tcp last attempt 311 00000400:00000200:3.0:1773941541.985885:0:12130:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:3.0:1773941541.985889:0:12130:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.143@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941541.985893:0:12130:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941541.985896:0:12130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941541.985898:0:12130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941541.985900:0:12130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000001:3.0:1773941541.985903:0:12130:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941541.985905:0:12130:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:3.0:1773941541.985907:0:12130:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941541.985909:0:12130:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:3.0:1773941541.985911:0:12130:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:3.0:1773941541.985914:0:12130:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00000040:3.0:1773941541.985917:0:12130:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941541.985919:0:12130:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941541.985921:0:12130:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b5ba039c900 refcount 3 to 192.168.204.143@tcp 00000100:00000001:3.0:1773941541.985924:0:12130:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615824379267328 : -128249330284288 : ffff8b5ba039c900) 00000020:00000040:3.0:1773941541.985928:0:12130:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00080000:3.0:1773941541.985931:0:12130:0:(import.c:623:import_select_connection()) MGC192.168.204.143@tcp: import ffff8b5b90f25800 using connection 192.168.204.143@tcp/192.168.204.143@tcp 00000100:00000001:3.0:1773941541.985935:0:12130:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941541.985938:0:12130:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941541.985942:0:12130:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941541.985945:0:12130:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.985949:0:12130:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941541.985951:0:12130:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941541.985956:0:12130:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90275180. 00000020:00000040:3.0:1773941541.985959:0:12130:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 02000000:00000001:3.0:1773941541.985964:0:12130:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941541.985965:0:12130:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.985967:0:12130:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941541.985969:0:12130:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941541.985986:0:12130:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941541.985991:0:12130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b5b90260c00. 00000100:00000001:3.0:1773941541.985999:0:12130:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941541.986005:0:12130:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b5b90275180 x1860112241169152/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941541.986012:0:12130:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941541.986013:0:12130:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941541.986017:0:12130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008177fcba] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90275180 x1860112241169152/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941541.986037:0:12130:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.986039:0:12130:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941541.986040:0:12130:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941541.986043:0:12130:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941541.986045:0:12130:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941541.986046:0:12130:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941541.986048:0:12130:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b5ba0b543c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941541.986050:0:12130:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941541.986051:0:12130:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941541.986055:0:12130:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b9021dc00 x1860112241169024/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941557 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941541.986060:0:12130:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941541.986063:0:12130:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941541.986065:0:12130:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941541.986067:0:12130:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941541.986070:0:12130:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b9021dc00 x1860112241169024/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941557 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941541.986076:0:12130:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b9021dc00 pname:cluuid:pid:xid:nid:opc:job mount.lustre:1080e6b5-6a6c-4762-b0ec-c785990fec17:12130:1860112241169024:192.168.204.143@tcp:101:mount.lustre.0 00000100:00000001:3.0:1773941541.986080:0:12130:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941541.986083:0:12130:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941541.986083:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941541.986088:0:12130:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941541.986090:0:12130:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941541.986090:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941541.986094:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941541.986095:0:12130:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b9021dc00 x1860112241169024/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941557 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941541.986103:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90275180 x1860112241169152/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941541.986117:0:12130:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.986121:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000010:3.0:1773941541.986122:0:12130:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff8b5ba0b543c0. 00000100:00000001:1.0:1773941541.986125:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.986127:0:12130:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:1.0:1773941541.986128:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1773941541.986130:0:12130:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:1.0:1773941541.986131:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1773941541.986136:0:12130:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:1.0:1773941541.986136:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941541.986142:0:12130:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00100000:1.0:1773941541.986142:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241169152:192.168.204.143@tcp:250:mount.lustre.0 00000020:00000001:3.0:1773941541.986145:0:12130:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941541.986149:0:12130:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5bc2150600 refcount=4 00000100:00000001:1.0:1773941541.986150:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1773941541.986152:0:12130:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824947283456 : -128248762268160 : ffff8b5bc2150600) 02000000:00000001:1.0:1773941541.986154:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:3.0:1773941541.986157:0:12130:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615824947283456 : -128248762268160 : ffff8b5bc2150600) 02000000:00000001:1.0:1773941541.986158:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941541.986163:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00010000:3.0:1773941541.986165:0:12130:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1e7b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:1.0:1773941541.986169:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b5ba0736800. 02000000:00000001:1.0:1773941541.986173:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941541.986176:0:12130:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:1.0:1773941541.986179:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00010000:3.0:1773941541.986183:0:12130:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1e7b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:1.0:1773941541.986184:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5b8441a9f8. 00000400:00000010:1.0:1773941541.986191:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f16e8. 00010000:00000001:3.0:1773941541.986193:0:12130:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000200:1.0:1773941541.986196:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112241169152, portal 25 00010000:00010000:3.0:1773941541.986197:0:12130:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1e7b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941541.986201:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941541.986204:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109621632 : -128249599929984 : ffff8b5b90275180) 00010000:00000001:3.0:1773941541.986208:0:12130:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941541.986213:0:12130:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000040:1.0:1773941541.986213:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90275180 x1860112241169152/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1773941541.986216:0:12130:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1e7b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941541.986234:0:12130:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941541.986236:0:12130:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.986238:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:3.0:1773941541.986241:0:12130:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000040:1.0:1773941541.986245:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00010000:00010000:3.0:1773941541.986246:0:12130:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1e7b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:1.0:1773941541.986253:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b854f13b8. 00000100:00000200:1.0:1773941541.986259:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112241169152, offset 0 00010000:00010000:3.0:1773941541.986264:0:12130:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b5bc2150600 already unused, calling callback (ffffffffc1482c20) 10000000:00000001:3.0:1773941541.986267:0:12130:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941541.986270:0:12130:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1e7b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941541.986279:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941541.986287:0:12130:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:3.0:1773941541.986290:0:12130:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941541.986292:0:12130:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1773941541.986293:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000020:00000040:3.0:1773941541.986294:0:12130:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff8b5bc2150600 refcount=5 00000020:00000001:3.0:1773941541.986299:0:12130:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824947283456 : -128248762268160 : ffff8b5bc2150600) 00010000:00000001:3.0:1773941541.986301:0:12130:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615824947283456 : -128248762268160 : ffff8b5bc2150600) 00000400:00000200:1.0:1773941541.986304:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00010000:00000001:3.0:1773941541.986305:0:12130:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:3.0:1773941541.986310:0:12130:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1e7b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941541.986312:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00000001:3.0:1773941541.986318:0:12130:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:1.0:1773941541.986319:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 10000000:00010000:3.0:1773941541.986321:0:12130:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1e7b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941541.986326:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:01000000:3.0:1773941541.986329:0:12130:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:1.0:1773941541.986333:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 10000000:00000040:3.0:1773941541.986334:0:12130:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941541.986338:0:12130:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941541.986342:0:12130:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:1.0:1773941541.986343:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[194, 195] 00010000:00000001:3.0:1773941541.986345:0:12130:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:3.0:1773941541.986350:0:12130:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:1.0:1773941541.986352:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.143@tcp 00010000:00000001:3.0:1773941541.986353:0:12130:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941541.986356:0:12130:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941541.986360:0:12130:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941541.986361:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [301:301:256:4294967295] 192.168.204.143@tcp LPNI seq info [196:196:8:4294967295] 00000020:00000040:3.0:1773941541.986363:0:12130:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5bc2150600 with handle 0xa33a5be14eae1e7b from hash 00010000:00000001:3.0:1773941541.986368:0:12130:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941541.986371:0:12130:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:1.0:1773941541.986371:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00010000:00000001:3.0:1773941541.986375:0:12130:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:1.0:1773941541.986381:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.143@tcp(192.168.204.143@tcp:192.168.204.143@tcp) : PUT try# 0 00010000:00000001:3.0:1773941541.986384:0:12130:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:3.0:1773941541.986387:0:12130:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941541.986391:0:12130:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000200:1.0:1773941541.986391:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.143@tcp 00010000:00000001:3.0:1773941541.986396:0:12130:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941541.986397:0:12130:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:1.0:1773941541.986398:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b9024bf00. 00010000:00000001:3.0:1773941541.986399:0:12130:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941541.986402:0:12130:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941541.986405:0:12130:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1e7b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:1.0:1773941541.986406:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b83284600] -> 12345-192.168.204.143@tcp (5) 00010000:00000001:3.0:1773941541.986414:0:12130:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941541.986415:0:12130:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941541.986419:0:12130:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000800:00000200:1.0:1773941541.986419:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.143@tcp ip 192.168.204.143:988 00010000:00000001:3.0:1773941541.986421:0:12130:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941541.986425:0:12130:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:1.0:1773941541.986425:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b9024bf00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:3.0:1773941541.986428:0:12130:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941541.986431:0:12130:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941541.986434:0:12130:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.143@tcp lock: ffff8b5bc2150600/0xa33a5be14eae1e7b lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 12130 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:3.0:1773941541.986439:0:12130:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8b5ba0d64500 count: 0 00000100:00000001:1.0:1773941541.986442:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941541.986447:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:1.0:1773941541.986448:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294982349 00000100:00000001:1.0:1773941541.986453:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294982347 00010000:00000001:3.0:1773941541.986454:0:12130:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941541.986457:0:12130:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941541.986459:0:12130:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941541.986463:0:12130:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b9021dc00 x1860112241169024/t0(0) o101->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941557 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.986464:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294982349 00000100:00000001:1.0:1773941541.986471:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294982347 00000100:00000001:3.0:1773941541.986473:0:12130:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1773941541.986475:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941541.986477:0:12130:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1773941541.986478:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941541.986481:0:12130:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0ce7000. 00000100:00000001:1.0:1773941541.986482:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941541.986486:0:12130:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941541.986486:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1773941541.986490:0:12130:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941541.986493:0:12130:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941541.986495:0:12130:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941541.986496:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000010:3.0:1773941541.986498:0:12130:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff8b5b90e7e400. 00000100:00000001:1.0:1773941541.986499:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:3.0:1773941541.986503:0:12130:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941541.986503:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1773941541.986506:0:12130:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941541.986513:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000010:3.0:1773941541.986516:0:12130:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b9021dc00. 00000100:00000001:1.0:1773941541.986517:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.986522:0:12130:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941541.986522:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.986526:0:12130:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941541.986528:0:12130:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941541.986530:0:12130:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941541.986533:0:12130:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941541.986537:0:12130:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff8b5b85068780. 00000020:00000010:3.0:1773941541.986542:0:12130:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab8600. 00000800:00000200:2.0:1773941541.986545:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000020:00000010:3.0:1773941541.986547:0:12130:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab8800. 00000800:00000010:2.0:1773941541.986550:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b9024bf00. 00000040:00000010:3.0:1773941541.986552:0:12130:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab9000. 00000400:00000200:2.0:1773941541.986557:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:3.0:1773941541.986559:0:12130:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941541.986564:0:12130:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab9800. 00000400:00000200:2.0:1773941541.986565:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 80000000:00000010:3.0:1773941541.986572:0:12130:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff8b5ba0bc9420. 00000400:00000200:2.0:1773941541.986572:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f13b8 00000400:00000010:2.0:1773941541.986574:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f13b8. 00000100:00000001:2.0:1773941541.986579:0:2347:0:(events.c:34:request_out_callback()) Process entered 00040000:00000010:3.0:1773941541.986580:0:12130:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab9e00. 00000100:00000200:2.0:1773941541.986584:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90275180 x1860112241169152/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 20000000:00000010:3.0:1773941541.986587:0:12130:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff8b5ba0ab9400. 10000000:00000040:3.0:1773941541.986594:0:12130:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000100:00000001:2.0:1773941541.986596:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00000001:3.0:1773941541.986598:0:12130:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:2.0:1773941541.986602:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90275180 x1860112241169152/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000020:00000010:3.0:1773941541.986605:0:12130:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab8800. 00000100:00000001:2.0:1773941541.986610:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941541.986612:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000040:00000010:3.0:1773941541.986670:0:12130:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab9000. 00000020:00000010:3.0:1773941541.986673:0:12130:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941541.986677:0:12130:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab9800. 80000000:00000010:3.0:1773941541.986679:0:12130:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9420. 00040000:00000010:3.0:1773941541.986683:0:12130:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab9e00. 20000000:00000010:3.0:1773941541.986685:0:12130:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff8b5ba0ab9400. 00000020:00000010:3.0:1773941541.986687:0:12130:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab8600. 10000000:00000010:3.0:1773941541.986690:0:12130:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff8b5b85068780. 10000000:01000000:3.0:1773941541.986692:0:12130:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.143@tcp: configuration from log 'lustre-client' failed (-107). 10000000:00000001:3.0:1773941541.986695:0:12130:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00000001:3.0:1773941541.986697:0:12130:0:(mgc_request.c:2026:mgc_process_config()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000020:00000010:3.0:1773941541.986700:0:12130:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b84d8ac00. 00000040:00000010:3.0:1773941541.986733:0:12130:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b84d8ba00. 00000020:00000010:3.0:1773941541.986736:0:12130:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b84e46000. 00000020:00000010:3.0:1773941541.986740:0:12130:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b84d8aa00. 00040000:00000010:3.0:1773941541.986749:0:12130:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98bb0d00. 00040000:00000010:3.0:1773941541.986752:0:12130:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b84d8a800. 00000020:00000010:3.0:1773941541.986755:0:12130:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84d8a600. 00000020:00000001:3.0:1773941541.986758:0:12130:0:(obd_class.h:696:obd_process_config()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000020:00000010:3.0:1773941541.986761:0:12130:0:(obd_mount.c:82:lustre_process_log()) kfreed 'lcfg': 184 at ffff8b5ba0b54540. 00000020:00000010:3.0:1773941541.986764:0:12130:0:(obd_mount.c:84:lustre_process_log()) kfreed 'bufs': 104 at ffff8b5b89390780. 00000020:02020000:3.0:1773941541.986767:0:12130:0:(obd_mount.c:91:lustre_process_log()) MGC192.168.204.143@tcp: Confguration from log lustre-client failed from MGS -107. Communication error between node & MGS, a bad configuration, or other errors. See syslog for more info 00000020:00000001:3.0:1773941541.990746:0:12130:0:(obd_mount.c:93:lustre_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000080:00000001:3.0:1773941541.990751:0:12130:0:(llite_lib.c:1489:ll_fill_super()) Process leaving via out_debugfs (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000800:00000001:2.0:1773941541.990777:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.990786:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941541.990788:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.990791:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941541.990795:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.990800:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.143@tcp : PUT - for me 00000400:00000200:2.0:1773941541.990805:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.990815:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.990819:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.990822:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 416 into portal 25 MB=0x69bc321609300 00000400:00000200:2.0:1773941541.990827:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.990832:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 416/416 into md 0x13fd [1] + 0 00000800:00000001:2.0:1773941541.990837:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.990845:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941541.990847:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941541.990852:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.143@tcp: PUT: OK 00000100:00000001:2.0:1773941541.990857:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941541.990863:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90275180 x1860112241169152/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941541.990877:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8b5b90275180 x1860112241169152/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941541.990905:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941541.990913:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.990919:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941541.990929:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941541.990932:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941541.990937:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941541.990939:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941541.990941:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a9f8. 00000400:00000200:1.0:1773941541.990945:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b854f16e8 00000400:00000010:1.0:1773941541.990947:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b854f16e8. 00000100:00000001:1.0:1773941541.990951:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941541.990956:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90275180 x1860112241169152/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941541.990966:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90275180 x1860112241169152/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.990981:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941541.990982:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941541.990984:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941541.990986:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941541.990988:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941541.990990:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941541.990992:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941541.990995:0:2351:0:(import.c:2136:obd_at_measure()) add 5 to ffff8b5b90f25c58 time=120 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941541.990998:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941541.991000:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941541.991002:0:2351:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941541.991004:0:2351:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941541.991006:0:2351:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.991009:0:2351:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941541.991013:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90275180 x1860112241169152/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.991018:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941541.991020:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941541.991022:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941541.991024:0:2351:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1773941541.991034:0:2351:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: connect to target with instance 0 00000020:00000001:1.0:1773941541.991037:0:2351:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941541.991038:0:2351:0:(genops.c:823:class_conn2export()) looking for export cookie 0xa33a5be14eae1c44 00000020:00000001:1.0:1773941541.991041:0:2351:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941541.991043:0:2351:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8b5ba0b0a800 refcount=3 00000020:00000001:1.0:1773941541.991045:0:2351:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000020:00000001:1.0:1773941541.991047:0:2351:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446615824387057664 : -128249322493952 : ffff8b5ba0b0a800) 00000100:00001000:1.0:1773941541.991052:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=1773941541 v=0 (0 0 0 0) 00000100:00001000:1.0:1773941541.991055:0:2351:0:(import.c:2202:obd_at_measure()) AT ffff8b5b90f25c10 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:1.0:1773941541.991062:0:2351:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 2 00000100:00000001:1.0:1773941541.991063:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941541.991065:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:1.0:1773941541.991067:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.991068:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941541.991073:0:2351:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.143@tcp after partition 00000100:00080000:1.0:1773941541.991075:0:2351:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.143@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:1.0:1773941541.991078:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:1.0:1773941541.991082:0:2351:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:1.0:1773941541.991086:0:2351:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.143@tcp; invalidating 00000020:00000040:1.0:1773941541.991089:0:2351:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=5 obd=MGC192.168.204.143@tcp 00000020:00000010:3.0:1773941541.991122:0:12130:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 576 at ffff8b5b83f0a400. 00000020:00000010:3.0:1773941541.991127:0:12130:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5b83f08c00. 00000020:00000010:3.0:1773941541.991130:0:12130:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 1440 at ffff8b5b84e43000. 00000020:00000010:3.0:1773941541.991136:0:12130:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5b83f08000. 00000080:00000010:3.0:1773941541.991139:0:12130:0:(llite_lib.c:1531:ll_fill_super()) kfreed 'cfg': 88 at ffff8b5ba0166a20. 00000080:00000001:3.0:1773941541.991144:0:12130:0:(llite_lib.c:1565:ll_put_super()) Process entered 00000080:00200000:3.0:1773941541.991146:0:12130:0:(llite_lib.c:1572:ll_put_super()) VFS Op: cfg_instance lustre-client-ffff8b5ba0b09800 (sb ffff8b5ba0b09800) 00000020:00000001:3.0:1773941541.991150:0:12130:0:(obd_mount.c:116:lustre_end_log()) Process entered 00000020:00000010:3.0:1773941541.991154:0:12130:0:(obd_mount.c:126:lustre_end_log()) kmalloced '(lcfg)': 176 at ffff8b5ba0b5e0c0. 00000020:00000001:3.0:1773941541.991156:0:12130:0:(obd_class.h:681:obd_process_config()) Process entered 00000020:00000010:3.0:1773941541.991158:0:12130:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab8600. 00000020:00000010:3.0:1773941541.991161:0:12130:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab9400. 00000040:00000010:3.0:1773941541.991163:0:12130:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab9e00. 00000020:00000010:3.0:1773941541.991166:0:12130:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941541.991168:0:12130:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab9800. 00040000:00000010:3.0:1773941541.991172:0:12130:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98a69d00. 00040000:00000010:3.0:1773941541.991174:0:12130:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab9000. 10000000:00000001:3.0:1773941541.991177:0:12130:0:(mgc_request.c:1932:mgc_process_config()) Process entered 10000000:00000001:3.0:1773941541.991180:0:12130:0:(mgc_request_server.c:668:mgc_process_config_server()) Process entered 10000000:00000001:3.0:1773941541.991183:0:12130:0:(mgc_request.c:493:config_log_end()) Process entered 10000000:00000001:3.0:1773941541.991185:0:12130:0:(mgc_request.c:159:config_log_find()) Process entered 10000000:00000001:3.0:1773941541.991187:0:12130:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941541.991189:0:12130:0:(mgc_request.c:108:config_log_get()) log lustre-client (ffff8b5b850b6c00) refs 2 10000000:00000001:3.0:1773941541.991190:0:12130:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941541.991198:0:12130:0:(mgc_request.c:177:config_log_find()) Process leaving (rc=18446615823923244032 : -128249786307584 : ffff8b5b850b6c00) 10000000:00000001:3.0:1773941541.991201:0:12130:0:(mgc_request.c:461:config_mark_cld_stop_nolock()) Process entered 10000000:00000040:3.0:1773941541.991202:0:12130:0:(mgc_request.c:467:config_mark_cld_stop_nolock()) lockh 0x0 00010000:00000001:3.0:1773941541.991205:0:12130:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:3.0:1773941541.991206:0:12130:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941541.991208:0:12130:0:(mgc_request.c:461:config_mark_cld_stop_nolock()) Process entered 10000000:00000040:3.0:1773941541.991209:0:12130:0:(mgc_request.c:467:config_mark_cld_stop_nolock()) lockh 0x0 00010000:00000001:3.0:1773941541.991211:0:12130:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:3.0:1773941541.991212:0:12130:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941541.991215:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941541.991216:0:12130:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff8b5b850b7c00) refs 1 10000000:01000000:3.0:1773941541.991218:0:12130:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-cliir 10000000:00000001:3.0:1773941541.991219:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991220:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991221:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991222:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991222:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991223:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991224:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991224:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991225:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991226:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 00000020:00000040:3.0:1773941541.991227:0:12130:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0c000 : new refcount 5 10000000:00000010:3.0:1773941541.991229:0:12130:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 285 at ffff8b5b850b7c00. 10000000:00000001:3.0:1773941541.991231:0:12130:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991233:0:12130:0:(mgc_request.c:461:config_mark_cld_stop_nolock()) Process entered 10000000:00000040:3.0:1773941541.991234:0:12130:0:(mgc_request.c:467:config_mark_cld_stop_nolock()) lockh 0x0 00010000:00000001:3.0:1773941541.991235:0:12130:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:3.0:1773941541.991236:0:12130:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941541.991237:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941541.991238:0:12130:0:(mgc_request.c:123:config_log_put()) log params(ffff8b5b850b6400) refs 1 10000000:01000000:3.0:1773941541.991240:0:12130:0:(mgc_request.c:130:config_log_put()) dropping config log params 10000000:00000001:3.0:1773941541.991241:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991242:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991242:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991243:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991243:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991244:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991245:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991249:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991250:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991251:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 00000020:00000040:3.0:1773941541.991253:0:12130:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0c000 : new refcount 4 10000000:00000010:3.0:1773941541.991254:0:12130:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 279 at ffff8b5b850b6400. 10000000:00000001:3.0:1773941541.991256:0:12130:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991258:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991259:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991259:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941541.991260:0:12130:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff8b5b84d8b600) refs 1 10000000:01000000:3.0:1773941541.991262:0:12130:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-sptlrpc 10000000:00000001:3.0:1773941541.991263:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991264:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991265:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991265:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991266:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991267:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991268:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991269:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991269:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991270:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 02000000:08000000:3.0:1773941541.991275:0:12130:0:(sec_config.c:500:sptlrpc_conf_free()) free sptlrpc conf lustre 02000000:00000010:3.0:1773941541.991277:0:12130:0:(sec_config.c:504:sptlrpc_conf_free()) kfreed 'conf': 120 at ffff8b5b89390600. 00000020:00000040:3.0:1773941541.991279:0:12130:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0c000 : new refcount 3 10000000:00000010:3.0:1773941541.991281:0:12130:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 287 at ffff8b5b84d8b600. 10000000:00000001:3.0:1773941541.991282:0:12130:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991283:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991284:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991284:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941541.991285:0:12130:0:(mgc_request.c:123:config_log_put()) log lustre-client(ffff8b5b850b6c00) refs 2 10000000:00000001:3.0:1773941541.991286:0:12130:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991287:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941541.991288:0:12130:0:(mgc_request.c:123:config_log_put()) log lustre-client(ffff8b5b850b6c00) refs 1 10000000:01000000:3.0:1773941541.991289:0:12130:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-client 10000000:00000001:3.0:1773941541.991290:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991290:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991291:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991293:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991294:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991295:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991299:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991300:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941541.991301:0:12130:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941541.991302:0:12130:0:(mgc_request.c:120:config_log_put()) Process leaving 00000020:00000040:3.0:1773941541.991303:0:12130:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0c000 : new refcount 2 10000000:00000010:3.0:1773941541.991305:0:12130:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 286 at ffff8b5b850b6c00. 10000000:00000001:3.0:1773941541.991307:0:12130:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:01000000:3.0:1773941541.991309:0:12130:0:(mgc_request.c:547:config_log_end()) end config log lustre-client (0) 10000000:00000001:3.0:1773941541.991310:0:12130:0:(mgc_request.c:548:config_log_end()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941541.991312:0:12130:0:(mgc_request.c:2026:mgc_process_config()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941541.991314:0:12130:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab9400. 00000040:00000010:3.0:1773941541.991316:0:12130:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab9e00. 00000020:00000010:3.0:1773941541.991318:0:12130:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941541.991319:0:12130:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab9800. 00040000:00000010:3.0:1773941541.991321:0:12130:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98a69d00. 00040000:00000010:3.0:1773941541.991323:0:12130:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab9000. 00000020:00000010:3.0:1773941541.991325:0:12130:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab8600. 00000020:00000001:3.0:1773941541.991326:0:12130:0:(obd_class.h:696:obd_process_config()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941541.991327:0:12130:0:(obd_mount.c:131:lustre_end_log()) kfreed 'lcfg': 176 at ffff8b5ba0b5e0c0. 00000020:00000001:3.0:1773941541.991329:0:12130:0:(obd_mount.c:132:lustre_end_log()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.991330:0:12130:0:(obd_mount.c:116:lustre_end_log()) Process entered 00000020:00000010:3.0:1773941541.991332:0:12130:0:(obd_mount.c:126:lustre_end_log()) kmalloced '(lcfg)': 168 at ffff8b5ba0b5e0c0. 00000020:00000001:3.0:1773941541.991334:0:12130:0:(obd_class.h:681:obd_process_config()) Process entered 00000020:00000010:3.0:1773941541.991335:0:12130:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab8600. 00000020:00000010:3.0:1773941541.991337:0:12130:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab9000. 00000040:00000010:3.0:1773941541.991339:0:12130:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab9800. 00000020:00000010:3.0:1773941541.991340:0:12130:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941541.991342:0:12130:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab9e00. 00040000:00000010:3.0:1773941541.991344:0:12130:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98a69d00. 00040000:00000010:3.0:1773941541.991345:0:12130:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab9400. 10000000:00000001:3.0:1773941541.991347:0:12130:0:(mgc_request.c:1932:mgc_process_config()) Process entered 10000000:00000001:3.0:1773941541.991348:0:12130:0:(mgc_request_server.c:668:mgc_process_config_server()) Process entered 10000000:00000001:3.0:1773941541.991349:0:12130:0:(mgc_request.c:493:config_log_end()) Process entered 10000000:00000001:3.0:1773941541.991350:0:12130:0:(mgc_request.c:159:config_log_find()) Process entered 10000000:00000001:3.0:1773941541.991352:0:12130:0:(mgc_request.c:177:config_log_find()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941541.991354:0:12130:0:(mgc_request.c:497:config_log_end()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 10000000:00000001:3.0:1773941541.991359:0:12130:0:(mgc_request.c:2026:mgc_process_config()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000020:00000010:3.0:1773941541.991361:0:12130:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab9000. 00000040:00000010:3.0:1773941541.991365:0:12130:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab9800. 00000020:00000010:3.0:1773941541.991367:0:12130:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941541.991368:0:12130:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab9e00. 00040000:00000010:3.0:1773941541.991371:0:12130:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98a69d00. 00000100:00000001:0.0:1773941541.991372:0:2351:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:3.0:1773941541.991373:0:12130:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab9400. 00000020:00000010:3.0:1773941541.991376:0:12130:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab8600. 00000020:00000001:3.0:1773941541.991380:0:12130:0:(obd_class.h:696:obd_process_config()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000001:0.0:1773941541.991380:0:2351:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941541.991386:0:12130:0:(obd_mount.c:131:lustre_end_log()) kfreed 'lcfg': 168 at ffff8b5ba0b5e0c0. 00000020:00000001:3.0:1773941541.991389:0:12130:0:(obd_mount.c:132:lustre_end_log()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000100:00000040:0.0:1773941541.991392:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90275180 x1860112241169152/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941551 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000080:02000400:3.0:1773941541.991394:0:12130:0:(llite_lib.c:1617:ll_put_super()) Unmounted lustre-client 00000100:00000001:1.0:1773941541.991399:0:12261:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:0.0:1773941541.991410:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90275180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241169152:192.168.204.143@tcp:250:mount.lustre.0 00000100:00080000:1.0:1773941541.991412:0:12261:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.143@tcp to MGS@192.168.204.143@tcp 00000100:00100000:1.0:1773941541.991417:0:12261:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941541.991429:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941541.991432:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941541.991437:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90275180 x1860112241169152/t0(0) o250->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941551 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.991439:0:12261:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941541.991443:0:12261:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:0.0:1773941541.991447:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:1.0:1773941541.991448:0:12261:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:0.0:1773941541.991450:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941541.991452:0:12261:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:0.0:1773941541.991453:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b5ba0736800. 00000001:00000001:1.0:1773941541.991457:0:12261:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941541.991458:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941541.991459:0:12261:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:0.0:1773941541.991461:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:1.0:1773941541.991462:0:12261:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941541.991465:0:12261:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941541.991465:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=4 obd=MGC192.168.204.143@tcp 10000000:00000001:1.0:1773941541.991469:0:12261:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941541.991471:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941541.991472:0:12261:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:0.0:1773941541.991474:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b5b90260c00. 00000100:00080000:1.0:1773941541.991477:0:12261:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:0.0:1773941541.991478:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941541.991479:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941541.991481:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90275180. 00000100:00000001:0.0:1773941541.991483:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941541.991484:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941541.991489:0:12261:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941541.991489:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941541.991493:0:12261:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941541.991495:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941541.991496:0:12261:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.991498:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941541.991500:0:12261:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941541.991502:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941541.991503:0:12261:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941541.991506:0:12261:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:0.0:1773941541.991506:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:1.0:1773941541.991510:0:12261:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.143@tcp: Reactivating import 00000100:00080000:1.0:1773941541.991525:0:12261:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941541.991527:0:12261:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.991528:0:12261:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941541.991531:0:12261:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.143@tcp: Connection restored to 192.168.204.143@tcp (at 192.168.204.143@tcp) 00000100:00000001:1.0:1773941541.991534:0:12261:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941541.991535:0:12261:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941541.991537:0:12261:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:1.0:1773941541.991539:0:12261:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941541.991540:0:12261:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.992224:0:12130:0:(obd_config.c:1231:class_del_profile()) Process entered 00000020:01000000:3.0:1773941541.992226:0:12130:0:(obd_config.c:1233:class_del_profile()) Del profile lustre-client 00000020:00000001:3.0:1773941541.992228:0:12130:0:(obd_config.c:1149:class_get_profile_nolock()) Process entered 00000020:00000001:3.0:1773941541.992230:0:12130:0:(obd_config.c:1156:class_get_profile_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.992232:0:12130:0:(obd_config.c:1248:class_del_profile()) Process leaving 00000080:00000001:3.0:1773941541.992234:0:12130:0:(llite_lib.c:266:ll_free_sbi()) Process entered 00000020:00000010:3.0:1773941541.992257:0:12130:0:(cl_page.c:1415:cl_cache_decref()) kfreed 'cache': 152 at ffff8b5ba0b54e40. 00000080:00000010:3.0:1773941541.992260:0:12130:0:(llite_lib.c:281:ll_free_sbi()) kfreed 'sbi->ll_foreign_symlink_prefix': 6 at ffff8b5b84f33490. 00000080:00000010:3.0:1773941541.992262:0:12130:0:(llite_lib.c:287:ll_free_sbi()) kfreed 'sbi->ll_foreign_symlink_upcall': 5 at ffff8b5b84f33de0. 00000080:00000010:3.0:1773941541.992287:0:12130:0:(llite_lib.c:310:ll_free_sbi()) kfreed 'sbi': 1376 at ffff8b5b84e42800. 00000080:00000001:3.0:1773941541.992290:0:12130:0:(llite_lib.c:312:ll_free_sbi()) Process leaving 00000020:00000001:3.0:1773941541.992292:0:12130:0:(obd_mount.c:909:lustre_common_put_super()) Process entered 00000020:01000004:3.0:1773941541.992294:0:12130:0:(obd_mount.c:911:lustre_common_put_super()) dropping sb ffff8b5ba0b09800 00000020:00000001:3.0:1773941541.992296:0:12130:0:(obd_mount.c:575:lustre_stop_mgc()) Process entered 00000020:00000001:3.0:1773941541.992298:0:12130:0:(obd_class.h:902:obd_disconnect()) Process entered 00010000:00000001:3.0:1773941541.992300:0:12130:0:(ldlm_lib.c:759:client_disconnect_export()) Process entered 00010000:00000040:3.0:1773941541.992303:0:12130:0:(ldlm_lib.c:772:client_disconnect_export()) disconnect MGC192.168.204.143@tcp - 1 00000100:00000001:3.0:1773941541.992305:0:12130:0:(pinger.c:388:ptlrpc_pinger_del_import()) Process entered 00000100:00080000:3.0:1773941541.992307:0:12130:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 1080e6b5-6a6c-4762-b0ec-c785990fec17->MGS 00000020:00000001:3.0:1773941541.992309:0:12130:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941541.992310:0:12130:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=2 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941541.992314:0:12130:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941541.992315:0:12130:0:(pinger.c:401:ptlrpc_pinger_del_import()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941541.992324:0:12130:0:(ldlm_request.c:2367:ldlm_cli_cancel_unused()) Process entered 00000001:00000001:3.0:1773941541.992326:0:12130:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941541.992327:0:12130:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941541.992329:0:12130:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941541.992331:0:12130:0:(ldlm_request.c:2379:ldlm_cli_cancel_unused()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941541.992333:0:12130:0:(ldlm_resource.c:1268:ldlm_namespace_free_prior()) Process entered 00010000:00000001:3.0:1773941541.992336:0:12130:0:(ldlm_resource.c:1207:__ldlm_namespace_free()) Process entered 00000001:00000001:3.0:1773941541.992337:0:12130:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941541.992338:0:12130:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941541.992339:0:12130:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941541.992340:0:12130:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941541.992340:0:12130:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941541.992341:0:12130:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941541.992343:0:12130:0:(ldlm_resource.c:1251:__ldlm_namespace_free()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941541.992344:0:12130:0:(ldlm_resource.c:1293:ldlm_namespace_free_prior()) Process leaving 00000100:00000001:3.0:1773941541.992346:0:12130:0:(import.c:1924:ptlrpc_disconnect_import()) Process entered 00000100:00000001:3.0:1773941541.992349:0:12130:0:(import.c:1858:ptlrpc_disconnect_import_async()) Process entered 00000100:00000001:3.0:1773941541.992351:0:12130:0:(import.c:1750:ptlrpc_disconnect_prep_req()) Process entered 00000100:00000010:3.0:1773941541.992353:0:12130:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b5b90274700. 00000020:00000040:3.0:1773941541.992355:0:12130:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000100:00000001:3.0:1773941541.992356:0:12130:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941541.992358:0:12130:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941541.992359:0:12130:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941541.992362:0:12130:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b5b98a69d00. 00000100:00000001:3.0:1773941541.992365:0:12130:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.992367:0:12130:0:(import.c:1788:ptlrpc_disconnect_prep_req()) Process leaving (rc=18446615824109618944 : -128249599932672 : ffff8b5b90274700) 00000100:00080000:3.0:1773941541.992369:0:12130:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from FULL to CONNECTING 00000100:00000001:3.0:1773941541.992372:0:12130:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941541.992374:0:12130:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941541.992378:0:12130:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2e4aa82] to pc [ptlrpcd_rcv+-1] req@ffff8b5b90274700 x1860112241169280/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941541.992393:0:12130:0:(import.c:1904:ptlrpc_disconnect_import_async()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.992450:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941541.992453:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941541.992455:0:2351:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941541.992461:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b5b90274700 x1860112241169280/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941541.992468:0:2351:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941541.992470:0:2351:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941541.992472:0:2351:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941541.992474:0:2351:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941541.992476:0:2351:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941541.992478:0:2351:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b5b90274700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241169280:192.168.204.143@tcp:251:mount.lustre.0 00000100:00000001:0.0:1773941541.992481:0:2351:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941541.992491:0:2351:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941541.992493:0:2351:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941541.992495:0:2351:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941541.992498:0:2351:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b5b845c4a00. 02000000:00000001:0.0:1773941541.992500:0:2351:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941541.992504:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000010:0.0:1773941541.992507:0:2351:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b5ba0c93948. 00000400:00000010:0.0:1773941541.992512:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c6e8. 00000100:00000200:0.0:1773941541.992515:0:2351:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112241169280, portal 25 00000100:00000001:0.0:1773941541.992517:0:2351:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941541.992518:0:2351:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446615824109618944 : -128249599932672 : ffff8b5b90274700) 00000100:00000040:0.0:1773941541.992522:0:2351:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8b5b90274700 x1860112241169280/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941541.992528:0:2351:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941541.992530:0:2351:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.143@tcp 00000400:00000010:0.0:1773941541.992532:0:2351:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b5b85e6c088. 00000100:00000200:0.0:1773941541.992534:0:2351:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112241169280, offset 0 00000400:00000200:0.0:1773941541.992538:0:2351:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.143@tcp 00000400:00000200:0.0:1773941541.992542:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:0.0:1773941541.992546:0:2351:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.143@tcp local destination 00000400:00000200:0.0:1773941541.992548:0:2351:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941541.992551:0:2351:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.43@tcp 00000400:00000200:0.0:1773941541.992553:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941541.992555:0:2351:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.43@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941541.992557:0:2351:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.143@tcp, 192.168.204.153@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[196, 195] 00000400:00000200:0.0:1773941541.992559:0:2351:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.153@tcp 00000400:00000200:0.0:1773941541.992562:0:2351:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.43@tcp NI seq info: [302:302:256:4294967295] 192.168.204.153@tcp LPNI seq info [197:197:8:4294967295] 00000400:00000200:0.0:1773941541.992564:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:0.0:1773941541.992569:0:2351:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp:) -> 192.168.204.153@tcp(192.168.204.143@tcp:192.168.204.153@tcp) : PUT try# 0 00000800:00000200:0.0:1773941541.992573:0:2351:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.153@tcp 00000800:00000010:0.0:1773941541.992577:0:2351:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b5b99101000. 00000800:00000200:0.0:1773941541.992580:0:2351:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b5b885de6c0] -> 12345-192.168.204.153@tcp (5) 00000800:00000200:0.0:1773941541.992589:0:2351:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.153@tcp ip 192.168.204.153:988 00000800:00000200:0.0:1773941541.992591:0:2351:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8b5b99101000 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1773941541.992603:0:2351:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941541.992606:0:2351:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000100:00000001:0.0:1773941541.992608:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 2 is 4294982355 00000100:00000001:0.0:1773941541.992611:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4294982349 00000100:00000001:0.0:1773941541.992614:0:2351:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS new deadline estimation for cpu 3 is 4294982355 00000100:00000001:0.0:1773941541.992618:0:2351:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.143@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4294982349 00000100:00000001:0.0:1773941541.992621:0:2351:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941541.992623:0:2351:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.992625:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.992627:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941541.992630:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941541.992631:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1773941541.992633:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941541.992639:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941541.992641:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.992642:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941541.992679:0:2347:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1773941541.992684:0:2347:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b5b99101000. 00000400:00000200:2.0:1773941541.992688:0:2347:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941541.992692:0:2347:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000400:00000200:2.0:1773941541.992695:0:2347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c088 00000400:00000010:2.0:1773941541.992696:0:2347:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c088. 00000100:00000001:2.0:1773941541.992699:0:2347:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1773941541.992726:0:2347:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b5b90274700 x1860112241169280/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941541.992734:0:2347:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941541.992738:0:2347:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b5b90274700 x1860112241169280/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941541.992744:0:2347:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941541.992746:0:2347:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:2.0:1773941541.993430:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.993435:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1773941541.993437:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.993439:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941541.993442:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941541.993445:0:2346:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.43@tcp(192.168.204.43@tcp) <- 192.168.204.153@tcp : PUT - for me 00000400:00000200:2.0:1773941541.993450:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941541.993452:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.153@tcp to cpt 1 00000400:00000200:2.0:1773941541.993455:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.993457:0:2346:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.143@tcp of length 224 into portal 25 MB=0x69bc321609380 00000400:00000200:2.0:1773941541.993460:0:2346:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.143@tcp to cpt 1 00000400:00000200:2.0:1773941541.993464:0:2346:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.143@tcp of length 224/224 into md 0x1405 [1] + 224 00000800:00000001:2.0:1773941541.993467:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.993472:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941541.993474:0:2346:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941541.993477:0:2346:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.43@tcp->192.168.204.153@tcp: PUT: OK 00000100:00000001:2.0:1773941541.993481:0:2346:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941541.993486:0:2346:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff8b5b90274700 x1860112241169280/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941541.993496:0:2346:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8b5b90274700 x1860112241169280/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941541.993514:0:2346:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1773941541.993520:0:2346:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:2.0:1773941541.993523:0:2346:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941541.993573:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941541.993576:0:2351:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941541.993579:0:2351:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941541.993580:0:2351:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941541.993582:0:2351:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93948. 00000400:00000200:0.0:1773941541.993586:0:2351:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85e6c6e8 00000400:00000010:0.0:1773941541.993587:0:2351:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8b5b85e6c6e8. 00000100:00000001:0.0:1773941541.993589:0:2351:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941541.993594:0:2351:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b5b90274700 x1860112241169280/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941541.993602:0:2351:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b5b90274700 x1860112241169280/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941541.993607:0:2351:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941541.993608:0:2351:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941541.993610:0:2351:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941541.993617:0:2351:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941541.993618:0:2351:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941541.993620:0:2351:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941541.993622:0:2351:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941541.993625:0:2351:0:(import.c:2136:obd_at_measure()) add 0 to ffff8b5b90f25c58 time=120 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941541.993627:0:2351:0:(import.c:2136:obd_at_measure()) add 1 to ffff8b5b90f25c10 time=0 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941541.993629:0:2351:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941541.993630:0:2351:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941541.993632:0:2351:0:(client.c:1695:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:0.0:1773941541.993636:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b5b90274700 x1860112241169280/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQU/200/0 rc -107/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941541.993641:0:2351:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941541.993642:0:2351:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941541.993643:0:2351:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941541.993645:0:2351:0:(import.c:65:import_set_state_nolock()) ffff8b5b90f25800 MGS: changing import state from CONNECTING to CLOSED 00000100:00000001:0.0:1773941541.993647:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941541.993648:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941541.993649:0:2351:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941541.993650:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.993651:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941541.993652:0:2351:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941541.993653:0:2351:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941541.993654:0:2351:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.993656:0:2351:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000040:0.0:1773941541.993667:0:2351:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b5b90274700 x1860112241169280/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 1 fl Interpret:RNQU/200/0 rc 0/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941541.993684:0:2351:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b5b90274700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:1080e6b5-6a6c-4762-b0ec-c785990fec17:2351:1860112241169280:192.168.204.143@tcp:251:mount.lustre.0 00000100:00000001:3.0:1773941541.993685:0:12130:0:(import.c:1967:ptlrpc_disconnect_import()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.993692:0:2351:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941541.993693:0:12130:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00000001:0.0:1773941541.993696:0:2351:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1773941541.993697:0:12130:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000040:0.0:1773941541.993700:0:2351:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b5b90274700 x1860112241169280/t0(0) o251->MGC192.168.204.143@tcp@192.168.204.143@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 1 fl Complete:RNQU/200/0 rc 0/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941541.993724:0:2351:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00010000:00000010:3.1:1773941541.993727:0:12130:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8b5ba0d64500. 00010000:00000010:3.1:1773941541.993734:0:12130:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8b5bc2150600. 00000100:00000001:3.0:1773941541.993744:0:12130:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:3.0:1773941541.993745:0:12130:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941541.993746:0:12130:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941541.993748:0:12130:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:0.0:1773941541.993749:0:2351:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 10000000:01000000:3.0:1773941541.993750:0:12130:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941541.993753:0:12130:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941541.993753:0:2351:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b5b845c4a00. 00000100:00000001:3.0:1773941541.993758:0:12130:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:0.0:1773941541.993760:0:2351:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:3.0:1773941541.993761:0:12130:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000001:0.0:1773941541.993763:0:2351:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941541.993764:0:12130:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000040:0.0:1773941541.993767:0:2351:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=2 obd=MGC192.168.204.143@tcp 10000000:01000000:3.0:1773941541.993768:0:12130:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941541.993771:0:12130:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:0.0:1773941541.993772:0:2351:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:3.0:1773941541.993773:0:12130:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:0.0:1773941541.993777:0:2351:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b5b98a69d00. 00000001:00000001:3.0:1773941541.993778:0:12130:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941541.993782:0:12130:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941541.993782:0:12130:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941541.993787:0:12130:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941541.993790:0:12130:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941541.993791:0:2351:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1773941541.993792:0:12130:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:0.0:1773941541.993794:0:2351:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00010000:00000001:3.0:1773941541.993796:0:12130:0:(ldlm_lib.c:818:client_disconnect_export()) Process leaving 00000100:00000010:0.0:1773941541.993797:0:2351:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b5b90274700. 00000020:00000001:3.0:1773941541.993800:0:12130:0:(genops.c:1462:class_disconnect()) Process entered 00000100:00000001:0.0:1773941541.993801:0:2351:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000020:00000080:3.0:1773941541.993804:0:12130:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0xa33a5be14eae1c44 00000100:00000001:0.0:1773941541.993804:0:2351:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941541.993806:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1773941541.993808:0:12130:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5ba0b0a800 with handle 0xa33a5be14eae1c44 from hash 00000100:00000001:0.0:1773941541.993810:0:2351:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941541.993813:0:2351:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.993816:0:12130:0:(genops.c:1934:obd_stale_export_put()) Process entered 00000100:00000001:0.0:1773941541.993816:0:2351:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941541.993818:0:2351:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941541.993819:0:12130:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 1 00000020:00000001:3.0:1773941541.993822:0:12130:0:(genops.c:1959:obd_stale_export_put()) Process leaving 00000020:00000040:3.0:1773941541.993824:0:12130:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0a800 : new refcount 0 00000020:00000080:3.0:1773941541.993826:0:12130:0:(genops.c:904:class_export_put()) final put ffff8b5ba0b0a800/1080e6b5-6a6c-4762-b0ec-c785990fec17 00000020:00000001:3.0:1773941541.993829:0:12130:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.993839:0:12130:0:(genops.c:1493:class_disconnect()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941541.993840:0:12130:0:(ldlm_lib.c:831:client_disconnect_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.993842:0:12130:0:(obd_class.h:914:obd_disconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.993844:0:12130:0:(obd_config.c:2489:class_manual_cleanup()) Process entered 00000020:01000000:3.0:1773941541.993846:0:12130:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of MGC192.168.204.143@tcp (flags='') 00000020:00000010:3.0:1773941541.993851:0:12130:0:(obd_config.c:2506:class_manual_cleanup()) kmalloced '(lcfg)': 72 at ffff8b5ba0bc9420. 00000020:00000080:3.0:1773941541.993853:0:12130:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000001:0.0:1773941541.993856:0:37:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000001:3.0:1773941541.993859:0:12130:0:(obd_config.c:936:class_cleanup()) Process entered 00000020:00000080:0.0:1773941541.993861:0:37:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5ba0b0a800/1080e6b5-6a6c-4762-b0ec-c785990fec17 for MGC192.168.204.143@tcp 00000020:00000080:3.0:1773941541.993864:0:12130:0:(obd_config.c:991:class_cleanup()) MGC192.168.204.143@tcp: forcing exports to disconnect: 0/3 00000020:00000040:0.0:1773941541.993867:0:37:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 2 to 192.168.204.143@tcp 00000020:00000001:0.0:1773941541.993869:0:37:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:0.0:1773941541.993873:0:37:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.993875:0:12130:0:(genops.c:1562:class_disconnect_exports()) Process entered 00000020:00000040:0.0:1773941541.993877:0:37:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.143@tcp (ffff8b5b982fb370) now 5 - export 00000020:00080000:3.0:1773941541.993879:0:12130:0:(genops.c:1577:class_disconnect_exports()) OBD device 0 (ffff8b5b982fb370) has no exports 00000020:00000010:0.0:1773941541.993882:0:37:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff8b5ba0b0a800. 00000020:00000001:3.0:1773941541.993883:0:12130:0:(genops.c:1578:class_disconnect_exports()) Process leaving 00000020:00000001:3.0:1773941541.993899:0:12130:0:(obd_class.h:604:obd_precleanup()) Process entered 00000020:00000010:3.0:1773941541.993902:0:12130:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab8600. 00000020:00000001:0.0:1773941541.993903:0:37:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000010:3.0:1773941541.993905:0:12130:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab9400. 00000040:00000010:3.0:1773941541.993909:0:12130:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab9e00. 00000020:00000010:3.0:1773941541.993911:0:12130:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941541.993914:0:12130:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab9800. 00040000:00000010:3.0:1773941541.993917:0:12130:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98a69d00. 00040000:00000010:3.0:1773941541.993919:0:12130:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab9000. 00000020:00000010:3.0:1773941541.993923:0:12130:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff8b5b85068780. 10000000:00000001:3.0:1773941541.993926:0:12130:0:(mgc_request.c:792:mgc_device_fini()) Process entered 10000000:00000001:2.0:1773941541.993961:0:12144:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941541.993964:0:12144:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:01000000:2.0:1773941541.993984:0:12144:0:(mgc_request.c:708:mgc_requeue_thread()) Ending requeue thread 10000000:00000001:2.0:1773941541.993986:0:12144:0:(mgc_request.c:709:mgc_requeue_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941541.993997:0:12130:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 10000000:01000000:3.0:1773941541.994000:0:12130:0:(obd_class.h:663:obd_cleanup_client_import()) MGC192.168.204.143@tcp: client import never connected 00000100:00000001:3.0:1773941541.994002:0:12130:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941541.994003:0:12130:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941541.994005:0:12130:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:3.0:1773941541.994007:0:12130:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941541.994008:0:12130:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941541.994009:0:12130:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941541.994010:0:12130:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941541.994012:0:12130:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.994014:0:12130:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00100000:3.0:1773941541.994015:0:12130:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941541.994016:0:12130:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941541.994017:0:12130:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941541.994019:0:12130:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941541.994020:0:12130:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941541.994023:0:12130:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941541.994024:0:12130:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941541.994025:0:12130:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941541.994026:0:12130:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941541.994028:0:12130:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.994029:0:12130:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:3.0:1773941541.994032:0:12130:0:(genops.c:1229:class_import_get()) import ffff8b5b90f25800 refcount=3 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941541.994034:0:12130:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941541.994039:0:12130:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=2 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941541.994041:0:12130:0:(genops.c:1249:class_import_put()) Process leaving 02000000:08000000:3.0:1773941541.994044:0:12130:0:(sec.c:1366:sec_cop_destroy_sec()) sec.null@ffffffffc1067ea0: being destroyed 00000020:00000001:3.0:1773941541.994046:0:12130:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941541.994047:0:12130:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=1 obd=MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941541.994048:0:12130:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000001:3.0:1773941541.994049:0:12130:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 10000000:00000001:3.0:1773941541.994051:0:12130:0:(mgc_request.c:774:mgc_llog_fini()) Process entered 10000000:00000040:3.0:1773941541.994052:0:12130:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff8b5ba0b54900 : new refcount 2 00000040:00000001:3.0:1773941541.994054:0:12130:0:(llog_obd.c:93:llog_cleanup()) Process entered 00000040:00000040:3.0:1773941541.994056:0:12130:0:(lustre_log.h:400:llog_ctxt_put()) PUTting ctxt ffff8b5ba0b54900 : new refcount 1 00000020:00000040:3.0:1773941541.994058:0:12130:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0c000 : new refcount 1 00000020:00000001:3.0:1773941541.994058:0:12130:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941541.994060:0:12130:0:(genops.c:1242:class_import_put()) import ffff8b5b90f25800 refcount=0 obd=MGC192.168.204.143@tcp 00000020:00000040:3.0:1773941541.994061:0:12130:0:(genops.c:1245:class_import_put()) final put import ffff8b5b90f25800 00000020:00000001:3.0:1773941541.994070:0:12130:0:(genops.c:1249:class_import_put()) Process leaving 00000040:00000010:3.0:1773941541.994071:0:12130:0:(llog_obd.c:46:llog_ctxt_destroy()) kfreed 'ctxt': 168 at ffff8b5ba0b54900. 00000040:00000001:3.0:1773941541.994076:0:12130:0:(llog_obd.c:122:llog_cleanup()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941541.994077:0:12130:0:(mgc_request.c:780:mgc_llog_fini()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941541.994079:0:12130:0:(mgc_request.c:816:mgc_device_fini()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941541.994084:0:12130:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068780. 00000020:00000010:3.0:1773941541.994092:0:12130:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab9400. 00000040:00000010:3.0:1773941541.994095:0:12130:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab9e00. 00000020:00000001:0.0:1773941541.994096:0:37:0:(genops.c:1200:obd_zombie_import_free()) Process entered 00000020:00000010:3.0:1773941541.994100:0:12130:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000020:00000080:0.0:1773941541.994100:0:37:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff8b5b90f25800 for MGC192.168.204.143@tcp 00000020:00000010:3.0:1773941541.994105:0:12130:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab9800. 00000020:00000040:0.0:1773941541.994106:0:37:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 1 to 192.168.204.143@tcp 00040000:00000010:3.0:1773941541.994109:0:12130:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98a69d00. 00000020:00000040:0.0:1773941541.994113:0:37:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8b5ba039c900 refcount 0 to 192.168.204.143@tcp 00040000:00000010:3.0:1773941541.994115:0:12130:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab9000. 00000020:00000010:0.0:1773941541.994117:0:37:0:(genops.c:1213:obd_zombie_import_free()) kfreed 'imp_conn': 88 at ffff8b5ba0166540. 00000020:00000010:3.0:1773941541.994119:0:12130:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab8600. 00000020:00000001:3.0:1773941541.994125:0:12130:0:(obd_class.h:623:obd_precleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941541.994125:0:37:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.143@tcp (ffff8b5b982fb370) now 4 - import 00000020:00000010:0.0:1773941541.994131:0:37:0:(genops.c:1220:obd_zombie_import_free()) kfreed 'imp': 1712 at ffff8b5b90f25800. 00000020:00000001:0.0:1773941541.994135:0:37:0:(genops.c:1221:obd_zombie_import_free()) Process leaving 00000001:00000001:3.0:1773941541.994140:0:12130:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:3.0:1773941541.994146:0:12130:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5b84feec00. 00000001:00000010:3.0:1773941541.994150:0:12130:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5b84fee200. 00000001:00000010:3.0:1773941541.994151:0:12130:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5b84feea00. 00000001:00000010:3.0:1773941541.994153:0:12130:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff8b5b84fefa00. 00000001:00000010:3.0:1773941541.994155:0:12130:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 32 at ffff8b5b83f0c9c0. 00000001:00000010:3.0:1773941541.994157:0:12130:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 232 at ffff8b5b99101e00. 00000001:00000001:3.0:1773941541.994163:0:12130:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00000020:00000040:3.0:1773941541.994165:0:12130:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.143@tcp (ffff8b5b982fb370) now 3 - setup 00000020:00000001:3.0:1773941541.994167:0:12130:0:(obd_config.c:1021:class_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.994169:0:12130:0:(obd_config.c:1659:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000080:3.0:1773941541.994171:0:12130:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000001:3.0:1773941541.994173:0:12130:0:(obd_config.c:899:class_detach()) Process entered 00000020:00000040:3.0:1773941541.994175:0:12130:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.143@tcp (ffff8b5b982fb370) now 2 - obd_device_list 00000020:00000080:3.0:1773941541.994178:0:12130:0:(obd_config.c:919:class_detach()) detach on obd MGC192.168.204.143@tcp (uuid 1080e6b5-6a6c-4762-b0ec-c785990fec17) 00000020:00000040:3.0:1773941541.994179:0:12130:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.143@tcp (ffff8b5b982fb370) now 1 - newdev 00000020:00000040:3.0:1773941541.994181:0:12130:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8b5ba0b0c000 with handle 0xa33a5be14eae1c3d from hash 00000020:00000040:3.0:1773941541.994183:0:12130:0:(genops.c:898:class_export_put()) PUTting export ffff8b5ba0b0c000 : new refcount 0 00000020:00000080:3.0:1773941541.994184:0:12130:0:(genops.c:904:class_export_put()) final put ffff8b5ba0b0c000/1080e6b5-6a6c-4762-b0ec-c785990fec17 00000020:00000001:3.0:1773941541.994186:0:12130:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.994187:0:12130:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:3.0:1773941541.994188:0:12130:0:(genops.c:863:class_export_destroy()) destroying export ffff8b5ba0b0c000/1080e6b5-6a6c-4762-b0ec-c785990fec17 for MGC192.168.204.143@tcp 00000020:00000001:3.0:1773941541.994190:0:12130:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:3.0:1773941541.994192:0:12130:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941541.994193:0:12130:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff8b5ba0b0c000. 00000020:00000001:3.0:1773941541.994196:0:12130:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000040:3.0:1773941541.994197:0:12130:0:(genops.c:437:class_free_dev()) Release obd device MGC192.168.204.143@tcp obd_type name = mgc 00000020:01000000:3.0:1773941541.994199:0:12130:0:(genops.c:440:class_free_dev()) finishing cleanup of obd MGC192.168.204.143@tcp (1080e6b5-6a6c-4762-b0ec-c785990fec17) 00000020:00000001:3.0:1773941541.994200:0:12130:0:(obd_class.h:633:obd_cleanup()) Process entered 00000020:00000010:3.0:1773941541.994202:0:12130:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8b5ba0ab8600. 00000020:00000010:3.0:1773941541.994204:0:12130:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8b5ba0ab9000. 00000040:00000010:3.0:1773941541.994206:0:12130:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff8b5ba0ab9800. 00000020:00000010:3.0:1773941541.994208:0:12130:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff8b5b90d94000. 00000020:00000010:3.0:1773941541.994210:0:12130:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff8b5ba0ab9e00. 00040000:00000010:3.0:1773941541.994212:0:12130:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff8b5b98a69d00. 00040000:00000010:3.0:1773941541.994213:0:12130:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff8b5ba0ab9400. 10000000:00000001:3.0:1773941541.994216:0:12130:0:(mgc_request.c:824:mgc_device_free()) Process entered 00000020:00000001:3.0:1773941541.994218:0:12130:0:(obd_config.c:1281:class_del_profiles()) Process entered 00000020:00000001:3.0:1773941541.994220:0:12130:0:(obd_config.c:1294:class_del_profiles()) Process leaving 00010000:00000001:3.0:1773941541.994296:0:12130:0:(ldlm_lib.c:631:client_obd_cleanup()) Process entered 00010000:00000001:3.0:1773941541.994302:0:12130:0:(ldlm_resource.c:1304:ldlm_namespace_free_post()) Process entered 00010000:00000001:3.0:1773941541.994305:0:12130:0:(ldlm_pool.c:931:ldlm_pool_fini()) Process entered 00000020:00000010:3.0:1773941541.994325:0:12130:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 448 at ffff8b5b84fef600. 00000020:00000010:3.0:1773941541.994328:0:12130:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 352 at ffff8b5b84d8b000. 00000020:00000010:3.0:1773941541.994330:0:12130:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5b83f09400. 00010000:00000001:3.0:1773941541.994349:0:12130:0:(ldlm_pool.c:934:ldlm_pool_fini()) Process leaving 00000020:00000010:3.0:1773941541.994357:0:12130:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff8b5b90edb500. 00000020:00000010:3.0:1773941541.994360:0:12130:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff8b5b89390280. 00000020:00000010:3.0:1773941541.994362:0:12130:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff8b5b9020a580. 00000020:00000010:3.0:1773941541.994365:0:12130:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 96 at ffff8b5ba0166180. 00000020:00000010:3.0:1773941541.994367:0:12130:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5b83f0ac00. 00000001:00000001:3.0:1773941541.994399:0:12130:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:3.0:1773941541.994401:0:12130:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 304 at ffff8b5b84d8b200. 00000001:00000010:3.0:1773941541.994402:0:12130:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 304 at ffff8b5b84d8a000. 00000001:00000010:3.0:1773941541.994405:0:12130:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 16 at ffff8b5b880e08c0. 00000001:00000010:3.0:1773941541.994406:0:12130:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 280 at ffff8b5b84d8b400. 00000001:00000001:3.0:1773941541.994408:0:12130:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00010000:00000010:3.0:1773941541.994410:0:12130:0:(ldlm_resource.c:1327:ldlm_namespace_free_post()) kfreed 'ns': 848 at ffff8b5b83f0b000. 00010000:00000001:3.0:1773941541.994412:0:12130:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:3.0:1773941541.994414:0:12130:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00010000:00000001:3.0:1773941541.994415:0:12130:0:(ldlm_resource.c:1329:ldlm_namespace_free_post()) Process leaving 00010000:00000001:3.0:1773941541.994417:0:12130:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 00010000:00000001:3.0:1773941541.994418:0:12130:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 00010000:00000001:3.0:1773941541.994419:0:12130:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:3.0:1773941541.994421:0:12130:0:(ldlm_lockd.c:3527:ldlm_cleanup()) Process entered 00000001:00000010:3.0:1773941541.994436:0:12130:0:(shrinker_debug.c:208:ll_shrinker_free()) kfreed 's': 88 at ffff8b5ba0166f00. 00000001:00000010:3.0:1773941541.994439:0:12130:0:(shrinker_debug.c:208:ll_shrinker_free()) kfreed 's': 88 at ffff8b5ba01663c0. 00000020:00000001:0.0:1773941541.994511:0:12142:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941541.994514:0:12142:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941541.994517:0:12142:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-1 blwis in pool) 00010000:00000001:0.0:1773941541.994526:0:12142:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:0.0:1773941541.994528:0:12142:0:(ldlm_lockd.c:2878:ldlm_bl_thread_blwi()) Process leaving (rc=2 : 2 : 2) 00000020:00000010:0.0:1773941541.994540:0:12142:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068240. 00000020:00000010:0.0:1773941541.994545:0:12142:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b98183400. 00000020:00000010:0.0:1773941541.994550:0:12142:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b8907e800. 00000020:00000010:0.0:1773941541.994556:0:12142:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b88498000. 00000020:00000010:0.0:1773941541.994571:0:12142:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b845d5c00. 80000000:00000010:0.0:1773941541.994575:0:12142:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9de0. 40000000:00000010:0.0:1773941541.994579:0:12142:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b85068700. 00100000:00000010:0.0:1773941541.994584:0:12142:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b90d97000. 00000020:00000001:1.0:1773941541.994586:0:12141:0:(genops.c:1914:obd_stale_export_get()) Process entered 00040000:00000010:0.0:1773941541.994587:0:12142:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98a69e00. 00000020:00000001:1.0:1773941541.994589:0:12141:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:0.0:1773941541.994592:0:12142:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b91e55200. 00010000:00010000:1.0:1773941541.994593:0:12141:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-2 blwis in pool) 00010000:00000001:1.0:1773941541.994597:0:12141:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:1.0:1773941541.994598:0:12141:0:(ldlm_lockd.c:2878:ldlm_bl_thread_blwi()) Process leaving (rc=2 : 2 : 2) 00002000:00000010:0.0:1773941541.994599:0:12142:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5b90260400. 00000020:00000010:0.0:1773941541.994603:0:12142:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b98183c00. 00010000:00000010:0.0:1773941541.994605:0:12142:0:(ldlm_lockd.c:3018:ldlm_bl_thread_main()) kfreed 'env': 56 at ffff8b5b85068940. 00010000:00000001:0.0:1773941541.994607:0:12142:0:(ldlm_lockd.c:3019:ldlm_bl_thread_main()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941541.994609:0:12141:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b845ad480. 00010000:00000010:3.0:1773941541.994623:0:12130:0:(ldlm_lockd.c:3556:ldlm_cleanup()) kfreed 'blp': 168 at ffff8b5ba0b54300. 00000020:00000010:1.0:1773941541.994623:0:12141:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b91df4600. 00000100:00000001:3.0:1773941541.994632:0:12130:0:(service.c:3841:ptlrpc_unregister_service()) Process entered 00000020:00000010:1.0:1773941541.994632:0:12141:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90f26800. 00000100:00000200:3.0:1773941541.994636:0:12130:0:(service.c:3843:ptlrpc_unregister_service()) ldlm_cbd: tearing down 00000020:00000010:1.0:1773941541.994640:0:12141:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b8855bc60. 00000100:00000001:3.0:1773941541.994643:0:12130:0:(service.c:3384:ptlrpc_stop_all_threads()) Process entered 00000100:00000001:3.0:1773941541.994646:0:12130:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 00000020:00000010:1.0:1773941541.994646:0:12141:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b84d8a400. 00000100:00000040:3.0:1773941541.994650:0:12130:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_cbd 80000000:00000010:1.0:1773941541.994652:0:12141:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0166360. 00000100:00000040:3.0:1773941541.994654:0:12130:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #1 40000000:00000010:1.0:1773941541.994656:0:12141:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b845ad2c0. 00000100:00000040:3.0:1773941541.994657:0:12130:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #0 00100000:00000010:1.0:1773941541.994661:0:12141:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b90f22800. 00040000:00000010:1.0:1773941541.994666:0:12141:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98bb0000. 00040000:00000010:1.0:1773941541.994669:0:12141:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b84d8b800. 00000100:00000040:3.0:1773941541.994672:0:12130:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cb #1 00002000:00000010:1.0:1773941541.994672:0:12141:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5b83f0b800. 00000020:00000010:1.0:1773941541.994676:0:12141:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b91df5800. 00010000:00000010:1.0:1773941541.994679:0:12141:0:(ldlm_lockd.c:3018:ldlm_bl_thread_main()) kfreed 'env': 56 at ffff8b5b845ad340. 00010000:00000001:1.0:1773941541.994683:0:12141:0:(ldlm_lockd.c:3019:ldlm_bl_thread_main()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941541.994685:0:12131:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b88909c00. 00000040:00000010:0.0:1773941541.994691:0:12131:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b88908600. 00000020:00000010:0.0:1773941541.994695:0:12131:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5ba0b0b000. 00000020:00000010:0.0:1773941541.994699:0:12131:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b98298f18. 00000020:00000010:0.0:1773941541.994729:0:12131:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b88908c00. 80000000:00000010:0.0:1773941541.994733:0:12131:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b85ef6000. 40000000:00000010:0.0:1773941541.994737:0:12131:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5ba039cf40. 00100000:00000010:0.0:1773941541.994741:0:12131:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b8a324800. 00040000:00000010:0.0:1773941541.994746:0:12131:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b99101300. 00040000:00000010:0.0:1773941541.994749:0:12131:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b8a327000. 00040000:00000010:0.0:1773941541.994751:0:12131:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b88908400. 00000004:00000010:0.0:1773941541.994756:0:12131:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b86885800. 00000004:00000010:0.0:1773941541.994774:0:12131:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5b98aca000. 00000004:00000010:0.0:1773941541.994781:0:12131:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b90db1800. 00002000:00000010:0.0:1773941541.994784:0:12131:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0bf0000. 00000004:00000010:0.0:1773941541.994786:0:12131:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5b98f0e000. 00000020:00000010:0.0:1773941541.994790:0:12131:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b8a0f2400. 00000020:00000010:0.0:1773941541.994794:0:12131:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5ba039c240. 00000100:00000010:0.0:1773941541.994798:0:12131:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5ba039c000. 00000100:00100000:0.0:1773941541.994805:0:12131:0:(service.c:3162:ptlrpc_main()) ldlm_cb00_000: service thread [ffff8b5b91fe9200:12131] 0 exiting: rc = 1 00000020:00000010:0.0:1773941541.994846:0:12132:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b88908000. 00000040:00000010:0.0:1773941541.994849:0:12132:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b88909a00. 00000020:00000010:0.0:1773941541.994851:0:12132:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90db0800. 00000020:00000010:0.0:1773941541.994854:0:12132:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b9829e4a0. 00000020:00000010:0.0:1773941541.994857:0:12132:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b88909800. 80000000:00000010:0.0:1773941541.994860:0:12132:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b85ef6300. 40000000:00000010:0.0:1773941541.994861:0:12132:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5ba039c080. 00100000:00000010:0.0:1773941541.994864:0:12132:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b90db5800. 00040000:00000010:0.0:1773941541.994866:0:12132:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b99101500. 00040000:00000010:0.0:1773941541.994868:0:12132:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b90db7000. 00040000:00000010:0.0:1773941541.994871:0:12132:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b88908a00. 00000004:00000010:0.0:1773941541.994873:0:12132:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b90db3000. 00000004:00000010:0.0:1773941541.994876:0:12132:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5b885ba000. 00000004:00000010:0.0:1773941541.994878:0:12132:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b90db7800. 00002000:00000010:0.0:1773941541.994880:0:12132:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5b98f0d800. 00000004:00000010:0.0:1773941541.994882:0:12132:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5b98f0c000. 00000020:00000010:0.0:1773941541.994897:0:12132:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b88909400. 00000020:00000010:0.0:1773941541.994899:0:12132:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5ba039c040. 00000100:00000010:0.0:1773941541.994901:0:12132:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5ba039c780. 00000100:00100000:0.0:1773941541.994902:0:12132:0:(service.c:3162:ptlrpc_main()) ldlm_cb00_001: service thread [ffff8b5b89317600:12132] 1 exiting: rc = 0 00000100:00000010:3.0:1773941541.994931:0:12130:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b91fe9200. 00000100:00000010:3.0:1773941541.994937:0:12130:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b89317600. 00000100:00000001:3.0:1773941541.994941:0:12130:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:3.0:1773941541.994943:0:12130:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 00000100:00000040:3.0:1773941541.994944:0:12130:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_cbd 00000100:00000040:3.0:1773941541.994946:0:12130:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #1 00000100:00000040:3.0:1773941541.994948:0:12130:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #0 00000100:00000040:3.0:1773941541.994980:0:12130:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cb #1 00000020:00000010:2.0:1773941541.995001:0:12133:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b8809ca00. 00000040:00000010:2.0:1773941541.995006:0:12133:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b8809ce00. 00000020:00000010:2.0:1773941541.995008:0:12133:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b98ac4000. 00000020:00000010:2.0:1773941541.995012:0:12133:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b88499420. 00000020:00000010:2.0:1773941541.995015:0:12133:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b8809c600. 80000000:00000010:2.0:1773941541.995023:0:12133:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9180. 40000000:00000010:2.0:1773941541.995026:0:12133:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b85068740. 00100000:00000010:2.0:1773941541.995028:0:12133:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b98ac2800. 00040000:00000010:2.0:1773941541.995032:0:12133:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98a69900. 00040000:00000010:2.0:1773941541.995035:0:12133:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b98ac3000. 00040000:00000010:2.0:1773941541.995038:0:12133:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b83991e00. 00000004:00000010:2.0:1773941541.995042:0:12133:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b98ac7000. 00000004:00000010:2.0:1773941541.995045:0:12133:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5ba0b88000. 00000004:00000010:2.0:1773941541.995048:0:12133:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b98ac0800. 00002000:00000010:2.0:1773941541.995051:0:12133:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5b90262c00. 00000004:00000010:2.0:1773941541.995053:0:12133:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5b90263c00. 00000020:00000010:2.0:1773941541.995055:0:12133:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b8809c800. 00000020:00000010:2.0:1773941541.995058:0:12133:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068400. 00000100:00000010:2.0:1773941541.995065:0:12133:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5b85068140. 00000100:00100000:2.0:1773941541.995067:0:12133:0:(service.c:3162:ptlrpc_main()) ldlm_cb01_000: service thread [ffff8b5b89316a00:12133] 0 exiting: rc = 1 00000020:00000010:2.0:1773941541.995101:0:12134:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b89316400. 00000040:00000010:2.0:1773941541.995105:0:12134:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b89317a00. 00000020:00000010:2.0:1773941541.995108:0:12134:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90f23800. 00000020:00000010:2.0:1773941541.995111:0:12134:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b88558000. 00000020:00000010:2.0:1773941541.995113:0:12134:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b89316000. 80000000:00000010:2.0:1773941541.995116:0:12134:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0166240. 40000000:00000010:2.0:1773941541.995118:0:12134:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b845ad180. 00100000:00000010:2.0:1773941541.995120:0:12134:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b90f25000. 00040000:00000010:2.0:1773941541.995122:0:12134:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98bb0400. 00040000:00000010:2.0:1773941541.995124:0:12134:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b90f24800. 00040000:00000010:2.0:1773941541.995126:0:12134:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b89316600. 00000004:00000010:2.0:1773941541.995128:0:12134:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b90f24000. 00000004:00000010:2.0:1773941541.995131:0:12134:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5b88905000. 00000004:00000010:2.0:1773941541.995133:0:12134:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b90f27800. 00002000:00000010:2.0:1773941541.995135:0:12134:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5b83f09000. 00000004:00000010:2.0:1773941541.995137:0:12134:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5b83f09c00. 00000020:00000010:2.0:1773941541.995139:0:12134:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b89317e00. 00000020:00000010:2.0:1773941541.995141:0:12134:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b845ada40. 00000100:00000010:2.0:1773941541.995143:0:12134:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5b845ad1c0. 00000100:00100000:2.0:1773941541.995145:0:12134:0:(service.c:3162:ptlrpc_main()) ldlm_cb01_001: service thread [ffff8b5b88909e00:12134] 1 exiting: rc = 0 00000100:00000010:3.0:1773941541.995176:0:12130:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b89316a00. 00000100:00000010:3.0:1773941541.995182:0:12130:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b88909e00. 00000100:00000001:3.0:1773941541.995185:0:12130:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:3.0:1773941541.995186:0:12130:0:(service.c:3391:ptlrpc_stop_all_threads()) Process leaving 00000400:00000200:3.0:1773941541.995190:0:12130:0:(lib-ptl.c:969:lnet_clear_lazy_portal()) clearing portal 15 lazy 00000400:00000001:3.0:1773941541.995194:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.995195:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.995198:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5ba0c93c08. 00000400:00000200:3.0:1773941541.995200:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b88ea7cc0 00000400:00000010:3.0:1773941541.995202:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b88ea7cc0. 00000100:00000001:3.0:1773941541.995205:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.995207:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_cbd 00000100:00100000:3.0:1773941541.995212:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.995213:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:3.0:1773941541.995215:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.995217:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.995218:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.995220:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87e18. 00000400:00000200:3.0:1773941541.995222:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90ec5540 00000400:00000010:3.0:1773941541.995223:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90ec5540. 00000100:00000001:3.0:1773941541.995225:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.995225:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_cbd 00000100:00100000:3.0:1773941541.995228:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.995230:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:3.0:1773941541.995231:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1773941541.995235:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.995236:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:3.0:1773941541.995242:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b850de000. 00000100:00000010:3.0:1773941541.995247:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0bf3400. 00000100:00000010:3.0:1773941541.995256:0:12130:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b90db4800. 00000100:00000010:3.0:1773941541.995258:0:12130:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b90db0000. 00000100:00000001:3.0:1773941541.995260:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.995261:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:3.0:1773941541.995263:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b850dc000. 00000100:00000010:3.0:1773941541.995265:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0bf1000. 00000100:00000010:3.0:1773941541.995267:0:12130:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b90f26000. 00000100:00000010:3.0:1773941541.995269:0:12130:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b98ac0000. 00000100:00000001:3.0:1773941541.995272:0:12130:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:3.0:1773941541.995274:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.995277:0:12130:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941541.995278:0:12130:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941541.995280:0:12130:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff8b5ba039ce00. 00000100:00000001:3.0:1773941541.995282:0:12130:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941541.995284:0:12130:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.995285:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.995288:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5ba0b6e0c0. 00000100:00000001:3.0:1773941541.995290:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.995291:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.995292:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5ba0b6ef00. 00000100:00000001:3.0:1773941541.995293:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.995294:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.995295:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5ba0b6e6c0. 00000100:00000001:3.0:1773941541.995296:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.995297:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.995298:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5ba0b6ee40. 00000100:00000001:3.0:1773941541.995299:0:12130:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000100:00000001:3.0:1773941541.995300:0:12130:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:3.0:1773941541.995301:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.995302:0:12130:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941541.995303:0:12130:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941541.995304:0:12130:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff8b5ba039c8c0. 00000100:00000001:3.0:1773941541.995305:0:12130:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941541.995306:0:12130:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.995307:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.995308:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b83f1f840. 00000100:00000001:3.0:1773941541.995310:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.995311:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.995311:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b83f1fa80. 00000100:00000001:3.0:1773941541.995316:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.995317:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.995318:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b845cf300. 00000100:00000001:3.0:1773941541.995320:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.995321:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.995322:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b845cf900. 00000100:00000001:3.0:1773941541.995323:0:12130:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000020:00000010:3.0:1773941541.995392:0:12130:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 3680 at ffff8b5b98acb000. 00000020:00000010:3.0:1773941541.995397:0:12130:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5ba0bf3c00. 00000100:00000010:3.0:1773941541.995415:0:12130:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff8b5b90d2c000. 00000100:00000010:3.0:1773941541.995423:0:12130:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff8b5b98acf000. 00000100:00000010:3.0:1773941541.995427:0:12130:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff8b5b98a98000. 00000100:00000010:3.0:1773941541.995433:0:12130:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff8b5b98ac8000. 00000100:00000010:3.0:1773941541.995436:0:12130:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff8b5ba0bf1400. 00000100:00000010:3.0:1773941541.995438:0:12130:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff8b5ba0bf1800. 00000100:00000010:3.0:1773941541.995440:0:12130:0:(service.c:3836:ptlrpc_service_free()) kfreed 'svc': 360 at ffff8b5b91fc7000. 00000100:00000001:3.0:1773941541.995443:0:12130:0:(service.c:3863:ptlrpc_unregister_service()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.995445:0:12130:0:(service.c:3841:ptlrpc_unregister_service()) Process entered 00000100:00000200:3.0:1773941541.995446:0:12130:0:(service.c:3843:ptlrpc_unregister_service()) ldlm_canceld: tearing down 00000100:00000001:3.0:1773941541.995449:0:12130:0:(service.c:3384:ptlrpc_stop_all_threads()) Process entered 00000100:00000001:3.0:1773941541.995450:0:12130:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 00000100:00000040:3.0:1773941541.995451:0:12130:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_canceld 00000100:00000040:3.0:1773941541.995453:0:12130:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #2 00000100:00000040:3.0:1773941541.995454:0:12130:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #1 00000100:00000040:3.0:1773941541.995455:0:12130:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #0 00000100:00000040:3.0:1773941541.995472:0:12130:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cn #2 00000020:00000010:0.0:1773941541.995533:0:12136:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b8a0d1800. 00000020:00000010:0.0:1773941541.995541:0:12136:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5b880f9660. 00000020:00000010:1.0:1773941541.995542:0:12135:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b902a0800. 00000040:00000010:0.0:1773941541.995546:0:12136:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b91d91400. 00000020:00000010:1.0:1773941541.995551:0:12135:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5b880f9a80. 00000020:00000010:0.0:1773941541.995553:0:12136:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b8896b000. 00000040:00000010:1.0:1773941541.995558:0:12135:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b902a1200. 00000020:00000010:0.0:1773941541.995561:0:12136:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b98a05f98. 00000020:00000010:1.0:1773941541.995565:0:12135:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b8896b800. 00000020:00000010:0.0:1773941541.995569:0:12136:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b91d90400. 00000020:00000010:1.0:1773941541.995572:0:12135:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b98a04670. 80000000:00000010:0.0:1773941541.995576:0:12136:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f98a0. 00000020:00000010:1.0:1773941541.995580:0:12135:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b902a0c00. 40000000:00000010:0.0:1773941541.995582:0:12136:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b850b0480. 80000000:00000010:1.0:1773941541.995586:0:12135:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f9ae0. 00008000:00000010:0.0:1773941541.995590:0:12136:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d26350. 40000000:00000010:1.0:1773941541.995592:0:12135:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b850b0140. 00100000:00000010:0.0:1773941541.995599:0:12136:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b8896d800. 00008000:00000010:1.0:1773941541.995600:0:12135:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d26a60. 00040000:00000010:0.0:1773941541.995607:0:12136:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b9024b700. 00100000:00000010:1.0:1773941541.995608:0:12135:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b88969800. 00040000:00000010:1.0:1773941541.995616:0:12135:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b9024bc00. 00040000:00000010:0.0:1773941541.995616:0:12136:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b8896f000. 00040000:00000010:1.0:1773941541.995621:0:12135:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b88968800. 00040000:00000010:0.0:1773941541.995624:0:12136:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b91d91800. 00040000:00000010:1.0:1773941541.995625:0:12135:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b902a0600. 00000004:00000010:1.0:1773941541.995628:0:12135:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b88969000. 00000004:00000010:1.0:1773941541.995630:0:12135:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5b90f35000. 00000004:00000010:1.0:1773941541.995634:0:12135:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b8896c000. 00002000:00000010:1.0:1773941541.995640:0:12135:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0734400. 00000004:00000010:0.0:1773941541.995642:0:12136:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b8896f800. 00000004:00000010:1.0:1773941541.995643:0:12135:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5ba0735000. 00000020:00000010:1.0:1773941541.995650:0:12135:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b902a1800. 00000004:00000010:0.0:1773941541.995650:0:12136:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5b90f32000. 00000020:00000010:1.0:1773941541.995654:0:12135:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b850b0740. 00000004:00000010:0.0:1773941541.995658:0:12136:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b8896c800. 00000100:00000010:1.0:1773941541.995659:0:12135:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5b850b0900. 00000100:00100000:1.0:1773941541.995666:0:12135:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_000: service thread [ffff8b5b845c4600:12135] 0 exiting: rc = 1 00002000:00000010:0.0:1773941541.995666:0:12136:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0736400. 00000004:00000010:0.0:1773941541.995673:0:12136:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5ba0737800. 00000020:00000010:0.0:1773941541.995677:0:12136:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b839e5e00. 00000020:00000010:0.0:1773941541.995688:0:12136:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b850b0bc0. 00000100:00000010:0.0:1773941541.995697:0:12136:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5b850b0b40. 00000100:00100000:0.0:1773941541.995702:0:12136:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_001: service thread [ffff8b5b89317000:12136] 1 exiting: rc = 0 00000020:00000010:1.0:1773941541.995738:0:12137:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b862a4400. 00000020:00000010:1.0:1773941541.995743:0:12137:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5b880f90c0. 00000040:00000010:1.0:1773941541.995747:0:12137:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b862a5400. 00000020:00000010:1.0:1773941541.995750:0:12137:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b8896e800. 00000020:00000010:1.0:1773941541.995754:0:12137:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b98a02d48. 00000020:00000010:1.0:1773941541.995757:0:12137:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b862a5000. 80000000:00000010:1.0:1773941541.995760:0:12137:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5b880f9cc0. 40000000:00000010:1.0:1773941541.995763:0:12137:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b850b0200. 00008000:00000010:1.0:1773941541.995767:0:12137:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d27c08. 00100000:00000010:1.0:1773941541.995770:0:12137:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b8896d000. 00040000:00000010:1.0:1773941541.995773:0:12137:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b9024b500. 00040000:00000010:1.0:1773941541.995776:0:12137:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b8896e000. 00040000:00000010:1.0:1773941541.995786:0:12137:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b862a4800. 00000004:00000010:1.0:1773941541.995790:0:12137:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b88968000. 00000004:00000010:1.0:1773941541.995793:0:12137:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5b90f31000. 00000004:00000010:1.0:1773941541.995795:0:12137:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b99ed1800. 00002000:00000010:1.0:1773941541.995798:0:12137:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5ba0735800. 00000004:00000010:1.0:1773941541.995800:0:12137:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5ba0736000. 00000020:00000010:1.0:1773941541.995802:0:12137:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b862a4e00. 00000020:00000010:1.0:1773941541.995805:0:12137:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b850b0e40. 00000100:00000010:1.0:1773941541.995808:0:12137:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5b850b0280. 00000100:00100000:1.0:1773941541.995810:0:12137:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_002: service thread [ffff8b5b89317400:12137] 2 exiting: rc = 0 00000100:00000010:3.0:1773941541.995835:0:12130:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b845c4600. 00000100:00000010:3.0:1773941541.995842:0:12130:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b89317000. 00000100:00000010:3.0:1773941541.995846:0:12130:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b89317400. 00000100:00000001:3.0:1773941541.995850:0:12130:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:3.0:1773941541.995851:0:12130:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 00000100:00000040:3.0:1773941541.995852:0:12130:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_canceld 00000100:00000040:3.0:1773941541.995854:0:12130:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #2 00000100:00000040:3.0:1773941541.995856:0:12130:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #1 00000100:00000040:3.0:1773941541.995857:0:12130:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #0 00000100:00000040:3.0:1773941541.995879:0:12130:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cn #2 00000020:00000010:3.0:1773941541.995889:0:12140:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b90c3da00. 00000020:00000010:3.0:1773941541.995894:0:12140:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9240. 00000020:00000010:2.0:1773941541.995895:0:12138:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba216bc00. 00000040:00000010:3.0:1773941541.995897:0:12140:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b90c3ce00. 00000020:00000010:2.0:1773941541.995903:0:12138:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9f00. 00000020:00000010:3.0:1773941541.995904:0:12140:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b98ac3800. 00000040:00000010:2.0:1773941541.995910:0:12138:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba216a000. 00000020:00000010:3.0:1773941541.995911:0:12140:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b8849f3b8. 00000020:00000010:2.0:1773941541.995915:0:12138:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b98ac2000. 00000020:00000010:3.0:1773941541.995918:0:12140:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b9a587600. 00000020:00000010:2.0:1773941541.995920:0:12138:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b88498a10. 80000000:00000010:3.0:1773941541.995923:0:12140:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc97e0. 00000020:00000010:2.0:1773941541.995925:0:12138:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba216be00. 40000000:00000010:3.0:1773941541.995928:0:12140:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b850686c0. 80000000:00000010:2.0:1773941541.995931:0:12138:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc9300. 00008000:00000010:3.0:1773941541.995933:0:12140:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b98a1c710. 40000000:00000010:2.0:1773941541.995936:0:12138:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b85068100. 00100000:00000010:3.0:1773941541.995939:0:12140:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b98ac6000. 00008000:00000010:2.0:1773941541.995941:0:12138:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b98a1fc08. 00040000:00000010:3.0:1773941541.995944:0:12140:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98a69100. 00100000:00000010:2.0:1773941541.995947:0:12138:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b98ac4800. 00040000:00000010:3.0:1773941541.995950:0:12140:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b98ac1800. 00040000:00000010:2.0:1773941541.995952:0:12138:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98a69a00. 00040000:00000010:3.0:1773941541.995955:0:12140:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b98182400. 00040000:00000010:2.0:1773941541.995957:0:12138:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b98ac7800. 00000004:00000010:3.0:1773941541.995963:0:12140:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b98ac5800. 00040000:00000010:2.0:1773941541.995964:0:12138:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b84ed5200. 00000004:00000010:3.0:1773941541.995987:0:12140:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5ba0b89000. 00000004:00000010:2.0:1773941541.995987:0:12138:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b98ac5000. 00000004:00000010:2.0:1773941541.995992:0:12138:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5ba0b8c000. 00000004:00000010:3.0:1773941541.995995:0:12140:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5ba21ce800. 00000004:00000010:2.0:1773941541.995997:0:12138:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b98ac6800. 00002000:00000010:3.0:1773941541.996000:0:12140:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5b90261400. 00002000:00000010:2.0:1773941541.996002:0:12138:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5b90263800. 00000004:00000010:3.0:1773941541.996005:0:12140:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5b90261800. 00000004:00000010:2.0:1773941541.996007:0:12138:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5b90260800. 00000020:00000010:3.0:1773941541.996010:0:12140:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b84ed4e00. 00000020:00000010:2.0:1773941541.996012:0:12138:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b99104800. 00000020:00000010:3.0:1773941541.996016:0:12140:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068c40. 00000020:00000010:2.0:1773941541.996019:0:12138:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b850680c0. 00000100:00000010:3.0:1773941541.996022:0:12140:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5b85068d80. 00000100:00000010:2.0:1773941541.996024:0:12138:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5b85068cc0. 00000100:00100000:3.0:1773941541.996027:0:12140:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_002: service thread [ffff8b5b84ed4a00:12140] 2 exiting: rc = 0 00000100:00100000:2.0:1773941541.996029:0:12138:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_000: service thread [ffff8b5b99105000:12138] 0 exiting: rc = 1 00000100:00000040:3.0:1773941541.996073:0:12130:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cn #1 00000020:00000010:2.0:1773941541.996090:0:12139:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b91df4800. 00000020:00000010:2.0:1773941541.996093:0:12139:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8b5ba0166ea0. 00000040:00000010:2.0:1773941541.996096:0:12139:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b91df5a00. 00000020:00000010:2.0:1773941541.996098:0:12139:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90f21800. 00000020:00000010:2.0:1773941541.996101:0:12139:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b8855e9a8. 00000020:00000010:2.0:1773941541.996104:0:12139:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b91df5e00. 80000000:00000010:2.0:1773941541.996106:0:12139:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0166720. 40000000:00000010:2.0:1773941541.996108:0:12139:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b845ad600. 00008000:00000010:2.0:1773941541.996110:0:12139:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8b5b90d774f8. 00100000:00000010:2.0:1773941541.996114:0:12139:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b90f20000. 00040000:00000010:2.0:1773941541.996116:0:12139:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98bb0f00. 00040000:00000010:2.0:1773941541.996118:0:12139:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b90f23000. 00040000:00000010:2.0:1773941541.996125:0:12139:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b91df5200. 00000004:00000010:2.0:1773941541.996127:0:12139:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b90f22000. 00000004:00000010:2.0:1773941541.996130:0:12139:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5b88906000. 00000004:00000010:2.0:1773941541.996132:0:12139:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b90f27000. 00002000:00000010:2.0:1773941541.996134:0:12139:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5b83f0b400. 00000004:00000010:2.0:1773941541.996136:0:12139:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5b83f0a800. 00000020:00000010:2.0:1773941541.996138:0:12139:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b91df4e00. 00000020:00000010:2.0:1773941541.996141:0:12139:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b845ad400. 00000100:00000010:2.0:1773941541.996143:0:12139:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff8b5b845ad8c0. 00000100:00100000:2.0:1773941541.996145:0:12139:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_001: service thread [ffff8b5b91df4200:12139] 1 exiting: rc = 0 00000100:00000010:3.0:1773941541.996162:0:12130:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b99105000. 00000100:00000010:3.0:1773941541.996171:0:12130:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b91df4200. 00000100:00000010:3.0:1773941541.996176:0:12130:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff8b5b84ed4a00. 00000100:00000001:3.0:1773941541.996180:0:12130:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:3.0:1773941541.996181:0:12130:0:(service.c:3391:ptlrpc_stop_all_threads()) Process leaving 00000400:00000200:3.0:1773941541.996184:0:12130:0:(lib-ptl.c:969:lnet_clear_lazy_portal()) clearing portal 17 lazy 00000400:00000001:3.0:1773941541.996188:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996189:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996191:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a630. 00000400:00000200:3.0:1773941541.996193:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b990e66c0 00000400:00000010:3.0:1773941541.996195:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b990e66c0. 00000100:00000001:3.0:1773941541.996202:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996203:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996207:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996208:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1773941541.996210:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996211:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996212:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996213:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a108. 00000400:00000200:3.0:1773941541.996215:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b990e6780 00000400:00000010:3.0:1773941541.996216:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b990e6780. 00000100:00000001:3.0:1773941541.996218:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996218:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996221:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996222:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 62 buffers still posted 00000100:00000001:3.0:1773941541.996223:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996224:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996225:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996226:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a790. 00000400:00000200:3.0:1773941541.996227:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b990e6600 00000400:00000010:3.0:1773941541.996228:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b990e6600. 00000100:00000001:3.0:1773941541.996229:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996230:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996233:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996234:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 61 buffers still posted 00000100:00000001:3.0:1773941541.996235:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996236:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996237:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996238:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a738. 00000400:00000200:3.0:1773941541.996239:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b990e6300 00000400:00000010:3.0:1773941541.996240:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b990e6300. 00000100:00000001:3.0:1773941541.996241:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996242:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996244:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996245:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 60 buffers still posted 00000100:00000001:3.0:1773941541.996246:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996247:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996248:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996249:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441acb8. 00000400:00000200:3.0:1773941541.996253:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b990e6480 00000400:00000010:3.0:1773941541.996254:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b990e6480. 00000100:00000001:3.0:1773941541.996256:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996257:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996259:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996260:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 59 buffers still posted 00000100:00000001:3.0:1773941541.996261:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996262:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996262:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996263:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441abb0. 00000400:00000200:3.0:1773941541.996265:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b990e6000 00000400:00000010:3.0:1773941541.996266:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b990e6000. 00000100:00000001:3.0:1773941541.996267:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996267:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996270:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996271:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 58 buffers still posted 00000100:00000001:3.0:1773941541.996272:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996272:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996273:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996274:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a268. 00000400:00000200:3.0:1773941541.996275:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b990e6cc0 00000400:00000010:3.0:1773941541.996276:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b990e6cc0. 00000100:00000001:3.0:1773941541.996277:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996278:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996280:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996281:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 57 buffers still posted 00000100:00000001:3.0:1773941541.996282:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996283:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996284:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996285:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a370. 00000400:00000200:3.0:1773941541.996286:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfa6c0 00000400:00000010:3.0:1773941541.996287:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfa6c0. 00000100:00000001:3.0:1773941541.996289:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996290:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996292:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996293:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 56 buffers still posted 00000100:00000001:3.0:1773941541.996294:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996298:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996299:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996300:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441ae70. 00000400:00000200:3.0:1773941541.996301:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfaf00 00000400:00000010:3.0:1773941541.996302:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfaf00. 00000100:00000001:3.0:1773941541.996303:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996304:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996306:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996307:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 55 buffers still posted 00000100:00000001:3.0:1773941541.996308:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996309:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996310:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996311:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a898. 00000400:00000200:3.0:1773941541.996312:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfacc0 00000400:00000010:3.0:1773941541.996312:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfacc0. 00000100:00000001:3.0:1773941541.996313:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996314:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996317:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996318:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 54 buffers still posted 00000100:00000001:3.0:1773941541.996318:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996319:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996320:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996320:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a580. 00000400:00000200:3.0:1773941541.996322:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfae40 00000400:00000010:3.0:1773941541.996322:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfae40. 00000100:00000001:3.0:1773941541.996323:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996324:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996327:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996328:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 53 buffers still posted 00000100:00000001:3.0:1773941541.996329:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996329:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996330:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996331:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441af20. 00000400:00000200:3.0:1773941541.996332:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfad80 00000400:00000010:3.0:1773941541.996333:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfad80. 00000100:00000001:3.0:1773941541.996334:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996335:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996337:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996341:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 52 buffers still posted 00000100:00000001:3.0:1773941541.996342:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996344:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996344:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996346:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441ab58. 00000400:00000200:3.0:1773941541.996347:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfa600 00000400:00000010:3.0:1773941541.996348:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfa600. 00000100:00000001:3.0:1773941541.996349:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996350:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996353:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996354:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 51 buffers still posted 00000100:00000001:3.0:1773941541.996354:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996355:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996356:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996357:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a5d8. 00000400:00000200:3.0:1773941541.996358:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfa300 00000400:00000010:3.0:1773941541.996359:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfa300. 00000100:00000001:3.0:1773941541.996360:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996360:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996363:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996364:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 50 buffers still posted 00000100:00000001:3.0:1773941541.996365:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996366:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996366:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996367:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a000. 00000400:00000200:3.0:1773941541.996368:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfa0c0 00000400:00000010:3.0:1773941541.996369:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfa0c0. 00000100:00000001:3.0:1773941541.996370:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996371:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996373:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996374:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 49 buffers still posted 00000100:00000001:3.0:1773941541.996375:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996376:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996376:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996377:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a8f0. 00000400:00000200:3.0:1773941541.996378:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfa480 00000400:00000010:3.0:1773941541.996380:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfa480. 00000100:00000001:3.0:1773941541.996381:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996385:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996388:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996389:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 48 buffers still posted 00000100:00000001:3.0:1773941541.996389:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996391:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996391:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996392:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441a840. 00000400:00000200:3.0:1773941541.996394:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfa240 00000400:00000010:3.0:1773941541.996394:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfa240. 00000100:00000001:3.0:1773941541.996396:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996396:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996399:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996400:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 47 buffers still posted 00000100:00000001:3.0:1773941541.996401:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996402:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996402:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996403:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b8441adc0. 00000400:00000200:3.0:1773941541.996404:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfa840 00000400:00000010:3.0:1773941541.996405:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfa840. 00000100:00000001:3.0:1773941541.996406:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996407:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996410:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996411:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 46 buffers still posted 00000100:00000001:3.0:1773941541.996411:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996413:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996413:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996414:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b850349f8. 00000400:00000200:3.0:1773941541.996416:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfac00 00000400:00000010:3.0:1773941541.996417:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfac00. 00000100:00000001:3.0:1773941541.996418:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996419:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996421:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996422:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 45 buffers still posted 00000100:00000001:3.0:1773941541.996423:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996424:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996425:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996426:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034d10. 00000400:00000200:3.0:1773941541.996427:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfa3c0 00000400:00000010:3.0:1773941541.996430:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfa3c0. 00000100:00000001:3.0:1773941541.996432:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996433:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996437:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996438:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 44 buffers still posted 00000100:00000001:3.0:1773941541.996439:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996440:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996441:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996442:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034948. 00000400:00000200:3.0:1773941541.996443:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfaa80 00000400:00000010:3.0:1773941541.996444:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfaa80. 00000100:00000001:3.0:1773941541.996445:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996445:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996448:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996449:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 43 buffers still posted 00000100:00000001:3.0:1773941541.996450:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996451:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996451:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996452:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034b00. 00000400:00000200:3.0:1773941541.996453:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfa540 00000400:00000010:3.0:1773941541.996454:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfa540. 00000100:00000001:3.0:1773941541.996455:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996456:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996459:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996460:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 42 buffers still posted 00000100:00000001:3.0:1773941541.996461:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996462:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996463:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996464:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034630. 00000400:00000200:3.0:1773941541.996465:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfa9c0 00000400:00000010:3.0:1773941541.996466:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfa9c0. 00000100:00000001:3.0:1773941541.996467:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996468:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996470:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996471:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 41 buffers still posted 00000100:00000001:3.0:1773941541.996472:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996473:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996476:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996478:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034108. 00000400:00000200:3.0:1773941541.996479:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfab40 00000400:00000010:3.0:1773941541.996480:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfab40. 00000100:00000001:3.0:1773941541.996482:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996483:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996486:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996487:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 40 buffers still posted 00000100:00000001:3.0:1773941541.996488:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996489:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996490:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996491:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034790. 00000400:00000200:3.0:1773941541.996492:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b84cfa900 00000400:00000010:3.0:1773941541.996493:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b84cfa900. 00000100:00000001:3.0:1773941541.996494:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996495:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996498:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996499:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 39 buffers still posted 00000100:00000001:3.0:1773941541.996499:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996501:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996501:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996502:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034738. 00000400:00000200:3.0:1773941541.996504:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0d99c00 00000400:00000010:3.0:1773941541.996505:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0d99c00. 00000100:00000001:3.0:1773941541.996506:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996507:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996510:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996511:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 38 buffers still posted 00000100:00000001:3.0:1773941541.996512:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996513:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996513:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996514:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034cb8. 00000400:00000200:3.0:1773941541.996515:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85299600 00000400:00000010:3.0:1773941541.996516:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85299600. 00000100:00000001:3.0:1773941541.996518:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996519:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996522:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996523:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 37 buffers still posted 00000100:00000001:3.0:1773941541.996526:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996528:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996528:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996529:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034bb0. 00000400:00000200:3.0:1773941541.996531:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b852996c0 00000400:00000010:3.0:1773941541.996532:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b852996c0. 00000100:00000001:3.0:1773941541.996533:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996533:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996536:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996537:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 36 buffers still posted 00000100:00000001:3.0:1773941541.996538:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996539:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996539:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996541:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034268. 00000400:00000200:3.0:1773941541.996542:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85299cc0 00000400:00000010:3.0:1773941541.996543:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85299cc0. 00000100:00000001:3.0:1773941541.996544:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996544:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996547:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996548:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 35 buffers still posted 00000100:00000001:3.0:1773941541.996549:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996550:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996550:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996551:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034370. 00000400:00000200:3.0:1773941541.996552:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b852999c0 00000400:00000010:3.0:1773941541.996554:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b852999c0. 00000100:00000001:3.0:1773941541.996555:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996555:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996568:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996569:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 34 buffers still posted 00000100:00000001:3.0:1773941541.996570:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996571:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996572:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996573:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034e70. 00000400:00000200:3.0:1773941541.996574:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b903fd600 00000400:00000010:3.0:1773941541.996575:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b903fd600. 00000100:00000001:3.0:1773941541.996577:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996577:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996583:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996584:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 33 buffers still posted 00000100:00000001:3.0:1773941541.996585:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996586:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996587:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996588:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034898. 00000400:00000200:3.0:1773941541.996589:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b903fd3c0 00000400:00000010:3.0:1773941541.996590:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b903fd3c0. 00000100:00000001:3.0:1773941541.996592:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996592:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996595:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996596:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 32 buffers still posted 00000100:00000001:3.0:1773941541.996597:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996598:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996598:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996599:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034580. 00000400:00000200:3.0:1773941541.996600:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b903fd540 00000400:00000010:3.0:1773941541.996601:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b903fd540. 00000100:00000001:3.0:1773941541.996602:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996603:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996606:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996607:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 31 buffers still posted 00000100:00000001:3.0:1773941541.996607:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996608:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996609:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996610:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034f20. 00000400:00000200:3.0:1773941541.996611:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b903fd000 00000400:00000010:3.0:1773941541.996612:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b903fd000. 00000100:00000001:3.0:1773941541.996613:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996613:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996616:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996617:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 30 buffers still posted 00000100:00000001:3.0:1773941541.996618:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996619:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996619:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996620:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034b58. 00000400:00000200:3.0:1773941541.996621:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b903fde40 00000400:00000010:3.0:1773941541.996622:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b903fde40. 00000100:00000001:3.0:1773941541.996626:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996627:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996631:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996632:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 29 buffers still posted 00000100:00000001:3.0:1773941541.996633:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996634:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996635:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996636:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b850345d8. 00000400:00000200:3.0:1773941541.996637:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b903fd240 00000400:00000010:3.0:1773941541.996638:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b903fd240. 00000100:00000001:3.0:1773941541.996639:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996640:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996643:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996643:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 28 buffers still posted 00000100:00000001:3.0:1773941541.996644:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996645:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996646:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996647:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034000. 00000400:00000200:3.0:1773941541.996648:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b903fdb40 00000400:00000010:3.0:1773941541.996649:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b903fdb40. 00000100:00000001:3.0:1773941541.996650:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996650:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996653:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996654:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 27 buffers still posted 00000100:00000001:3.0:1773941541.996655:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996656:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996657:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996658:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b850348f0. 00000400:00000200:3.0:1773941541.996659:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b903fd900 00000400:00000010:3.0:1773941541.996660:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b903fd900. 00000100:00000001:3.0:1773941541.996661:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996661:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996664:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996665:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 26 buffers still posted 00000100:00000001:3.0:1773941541.996666:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996667:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996667:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996668:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034840. 00000400:00000200:3.0:1773941541.996672:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b903fd300 00000400:00000010:3.0:1773941541.996673:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b903fd300. 00000100:00000001:3.0:1773941541.996674:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996675:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996678:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996679:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 25 buffers still posted 00000100:00000001:3.0:1773941541.996680:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996681:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996681:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996682:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034dc0. 00000400:00000200:3.0:1773941541.996683:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b903fdc00 00000400:00000010:3.0:1773941541.996684:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b903fdc00. 00000100:00000001:3.0:1773941541.996685:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996686:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996689:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996689:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 24 buffers still posted 00000100:00000001:3.0:1773941541.996690:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996691:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996692:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996693:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034c08. 00000400:00000200:3.0:1773941541.996694:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b903fda80 00000400:00000010:3.0:1773941541.996695:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b903fda80. 00000100:00000001:3.0:1773941541.996696:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996697:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996699:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996700:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 23 buffers still posted 00000100:00000001:3.0:1773941541.996701:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996864:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996865:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996866:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034d68. 00000400:00000200:3.0:1773941541.996869:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b91c56d80 00000400:00000010:3.0:1773941541.996871:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b91c56d80. 00000100:00000001:3.0:1773941541.996873:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996873:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996877:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996878:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 22 buffers still posted 00000100:00000001:3.0:1773941541.996879:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996885:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996886:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996887:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034ec8. 00000400:00000200:3.0:1773941541.996889:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85028480 00000400:00000010:3.0:1773941541.996890:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85028480. 00000100:00000001:3.0:1773941541.996892:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996893:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996896:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996897:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 21 buffers still posted 00000100:00000001:3.0:1773941541.996897:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996899:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996899:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996900:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b850347e8. 00000400:00000200:3.0:1773941541.996901:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85028d80 00000400:00000010:3.0:1773941541.996902:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85028d80. 00000100:00000001:3.0:1773941541.996903:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996904:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996907:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996908:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 20 buffers still posted 00000100:00000001:3.0:1773941541.996909:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996910:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996910:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996911:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b850349a0. 00000400:00000200:3.0:1773941541.996912:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85028c00 00000400:00000010:3.0:1773941541.996913:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85028c00. 00000100:00000001:3.0:1773941541.996914:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996915:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996918:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996919:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 19 buffers still posted 00000100:00000001:3.0:1773941541.996919:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996920:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996921:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996922:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b850343c8. 00000400:00000200:3.0:1773941541.996923:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b850280c0 00000400:00000010:3.0:1773941541.996924:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b850280c0. 00000100:00000001:3.0:1773941541.996925:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996926:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996928:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996932:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 18 buffers still posted 00000100:00000001:3.0:1773941541.996933:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996934:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996935:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996936:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034210. 00000400:00000200:3.0:1773941541.996937:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85028600 00000400:00000010:3.0:1773941541.996938:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85028600. 00000100:00000001:3.0:1773941541.996939:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996940:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996942:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996943:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 17 buffers still posted 00000100:00000001:3.0:1773941541.996944:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996945:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996946:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996947:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034c60. 00000400:00000200:3.0:1773941541.996948:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85028300 00000400:00000010:3.0:1773941541.996949:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85028300. 00000100:00000001:3.0:1773941541.996950:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996951:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996953:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996954:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 16 buffers still posted 00000100:00000001:3.0:1773941541.996955:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996956:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996956:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996957:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034aa8. 00000400:00000200:3.0:1773941541.996958:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85028000 00000400:00000010:3.0:1773941541.996959:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85028000. 00000100:00000001:3.0:1773941541.996960:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996961:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996964:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996965:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 15 buffers still posted 00000100:00000001:3.0:1773941541.996965:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996966:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996967:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996968:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034e18. 00000400:00000200:3.0:1773941541.996969:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b850289c0 00000400:00000010:3.0:1773941541.996970:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b850289c0. 00000100:00000001:3.0:1773941541.996985:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.996988:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.996992:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.996993:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 14 buffers still posted 00000100:00000001:3.0:1773941541.996993:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.996995:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.996996:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.996997:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034058. 00000400:00000200:3.0:1773941541.996998:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85028780 00000400:00000010:3.0:1773941541.996999:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85028780. 00000100:00000001:3.0:1773941541.997000:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997001:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997003:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997004:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 13 buffers still posted 00000100:00000001:3.0:1773941541.997005:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997006:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997007:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997008:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b850346e0. 00000400:00000200:3.0:1773941541.997009:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85028240 00000400:00000010:3.0:1773941541.997010:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85028240. 00000100:00000001:3.0:1773941541.997011:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997012:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997015:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997016:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 12 buffers still posted 00000100:00000001:3.0:1773941541.997017:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997018:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997019:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997019:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b850344d0. 00000400:00000200:3.0:1773941541.997021:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85028f00 00000400:00000010:3.0:1773941541.997022:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85028f00. 00000100:00000001:3.0:1773941541.997023:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997024:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997026:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997027:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 11 buffers still posted 00000100:00000001:3.0:1773941541.997028:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997029:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997030:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997031:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034688. 00000400:00000200:3.0:1773941541.997032:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b885deb40 00000400:00000010:3.0:1773941541.997036:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b885deb40. 00000100:00000001:3.0:1773941541.997038:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997039:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997043:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997044:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 10 buffers still posted 00000100:00000001:3.0:1773941541.997045:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997046:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997047:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997048:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b850340b0. 00000400:00000200:3.0:1773941541.997049:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b885de000 00000400:00000010:3.0:1773941541.997050:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b885de000. 00000100:00000001:3.0:1773941541.997051:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997052:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997055:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997055:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 9 buffers still posted 00000100:00000001:3.0:1773941541.997056:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997057:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997058:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997059:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b850342c0. 00000400:00000200:3.0:1773941541.997060:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b885de240 00000400:00000010:3.0:1773941541.997061:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b885de240. 00000100:00000001:3.0:1773941541.997062:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997063:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997065:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997066:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 8 buffers still posted 00000100:00000001:3.0:1773941541.997067:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997068:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997069:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997070:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034420. 00000400:00000200:3.0:1773941541.997071:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b885de540 00000400:00000010:3.0:1773941541.997072:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b885de540. 00000100:00000001:3.0:1773941541.997073:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997073:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997076:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997077:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 7 buffers still posted 00000100:00000001:3.0:1773941541.997078:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997079:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997079:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997083:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034f78. 00000400:00000200:3.0:1773941541.997085:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b885de3c0 00000400:00000010:3.0:1773941541.997086:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b885de3c0. 00000100:00000001:3.0:1773941541.997088:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997089:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997092:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997093:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 6 buffers still posted 00000100:00000001:3.0:1773941541.997093:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997094:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997095:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997096:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034160. 00000400:00000200:3.0:1773941541.997097:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b885de480 00000400:00000010:3.0:1773941541.997098:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b885de480. 00000100:00000001:3.0:1773941541.997099:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997100:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997103:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997104:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 5 buffers still posted 00000100:00000001:3.0:1773941541.997104:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997105:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997106:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997107:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034528. 00000400:00000200:3.0:1773941541.997108:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b885de0c0 00000400:00000010:3.0:1773941541.997109:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b885de0c0. 00000100:00000001:3.0:1773941541.997110:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997111:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997113:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997114:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 4 buffers still posted 00000100:00000001:3.0:1773941541.997115:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997116:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997117:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997117:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034318. 00000400:00000200:3.0:1773941541.997119:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b885ded80 00000400:00000010:3.0:1773941541.997120:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b885ded80. 00000100:00000001:3.0:1773941541.997121:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997121:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997124:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997125:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 3 buffers still posted 00000100:00000001:3.0:1773941541.997126:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997130:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997130:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997131:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b850341b8. 00000400:00000200:3.0:1773941541.997132:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b885de180 00000400:00000010:3.0:1773941541.997133:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b885de180. 00000100:00000001:3.0:1773941541.997134:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997135:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997138:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997139:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 2 buffers still posted 00000100:00000001:3.0:1773941541.997140:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997140:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997141:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997142:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034478. 00000400:00000200:3.0:1773941541.997143:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b885def00 00000400:00000010:3.0:1773941541.997144:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b885def00. 00000100:00000001:3.0:1773941541.997145:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997145:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997148:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997149:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 1 buffers still posted 00000100:00000001:3.0:1773941541.997150:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997151:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997151:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997152:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85034a50. 00000400:00000200:3.0:1773941541.997153:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b885decc0 00000400:00000010:3.0:1773941541.997154:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b885decc0. 00000100:00000001:3.0:1773941541.997155:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997156:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997158:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997159:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:3.0:1773941541.997160:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997162:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997163:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997164:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab688. 00000400:00000200:3.0:1773941541.997165:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e900 00000400:00000010:3.0:1773941541.997167:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e900. 00000100:00000001:3.0:1773941541.997168:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997169:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997171:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997175:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1773941541.997176:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997177:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997178:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997179:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab0b0. 00000400:00000200:3.0:1773941541.997180:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5eb40 00000400:00000010:3.0:1773941541.997181:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5eb40. 00000100:00000001:3.0:1773941541.997182:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997183:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997185:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997186:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 62 buffers still posted 00000100:00000001:3.0:1773941541.997187:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997188:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997189:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997190:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab2c0. 00000400:00000200:3.0:1773941541.997191:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5ecc0 00000400:00000010:3.0:1773941541.997192:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5ecc0. 00000100:00000001:3.0:1773941541.997193:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997193:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997196:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997197:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 61 buffers still posted 00000100:00000001:3.0:1773941541.997198:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997199:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997200:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997201:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab420. 00000400:00000200:3.0:1773941541.997202:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5ea80 00000400:00000010:3.0:1773941541.997203:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5ea80. 00000100:00000001:3.0:1773941541.997204:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997205:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997207:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997208:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 60 buffers still posted 00000100:00000001:3.0:1773941541.997209:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997210:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997210:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997211:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abf78. 00000400:00000200:3.0:1773941541.997212:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e840 00000400:00000010:3.0:1773941541.997213:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e840. 00000100:00000001:3.0:1773941541.997214:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997218:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997222:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997223:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 59 buffers still posted 00000100:00000001:3.0:1773941541.997224:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997226:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997227:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997229:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab160. 00000400:00000200:3.0:1773941541.997230:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e540 00000400:00000010:3.0:1773941541.997231:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e540. 00000100:00000001:3.0:1773941541.997232:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997232:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997235:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997236:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 58 buffers still posted 00000100:00000001:3.0:1773941541.997237:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997238:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997239:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997240:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab528. 00000400:00000200:3.0:1773941541.997241:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e180 00000400:00000010:3.0:1773941541.997242:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e180. 00000100:00000001:3.0:1773941541.997243:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997244:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997247:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997247:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 57 buffers still posted 00000100:00000001:3.0:1773941541.997248:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997249:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997250:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997251:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab318. 00000400:00000200:3.0:1773941541.997252:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e000 00000400:00000010:3.0:1773941541.997253:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e000. 00000100:00000001:3.0:1773941541.997254:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997255:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997257:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997258:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 56 buffers still posted 00000100:00000001:3.0:1773941541.997259:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997260:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997261:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997262:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab1b8. 00000400:00000200:3.0:1773941541.997263:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e3c0 00000400:00000010:3.0:1773941541.997267:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e3c0. 00000100:00000001:3.0:1773941541.997268:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997269:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997273:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997274:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 55 buffers still posted 00000100:00000001:3.0:1773941541.997275:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997276:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997277:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997278:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab478. 00000400:00000200:3.0:1773941541.997279:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5ee40 00000400:00000010:3.0:1773941541.997280:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5ee40. 00000100:00000001:3.0:1773941541.997281:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997282:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997284:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997285:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 54 buffers still posted 00000100:00000001:3.0:1773941541.997286:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997287:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997288:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997289:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845aba50. 00000400:00000200:3.0:1773941541.997290:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e780 00000400:00000010:3.0:1773941541.997291:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e780. 00000100:00000001:3.0:1773941541.997292:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997292:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997295:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997296:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 53 buffers still posted 00000100:00000001:3.0:1773941541.997297:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997298:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997299:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997300:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abc08. 00000400:00000200:3.0:1773941541.997301:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5ef00 00000400:00000010:3.0:1773941541.997302:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5ef00. 00000100:00000001:3.0:1773941541.997303:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997304:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997306:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997307:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 52 buffers still posted 00000100:00000001:3.0:1773941541.997308:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997309:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997312:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997313:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab948. 00000400:00000200:3.0:1773941541.997315:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5ec00 00000400:00000010:3.0:1773941541.997316:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5ec00. 00000100:00000001:3.0:1773941541.997317:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997318:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997321:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997322:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 51 buffers still posted 00000100:00000001:3.0:1773941541.997322:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997323:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997324:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997325:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abd68. 00000400:00000200:3.0:1773941541.997326:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5ba0b5e6c0 00000400:00000010:3.0:1773941541.997327:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5ba0b5e6c0. 00000100:00000001:3.0:1773941541.997328:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997329:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997331:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997332:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 50 buffers still posted 00000100:00000001:3.0:1773941541.997333:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997334:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997334:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997335:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abd10. 00000400:00000200:3.0:1773941541.997336:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b98124f00 00000400:00000010:3.0:1773941541.997337:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b98124f00. 00000100:00000001:3.0:1773941541.997339:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997339:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997342:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997343:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 49 buffers still posted 00000100:00000001:3.0:1773941541.997344:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997345:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997345:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997346:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab9f8. 00000400:00000200:3.0:1773941541.997347:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b98124a80 00000400:00000010:3.0:1773941541.997348:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b98124a80. 00000100:00000001:3.0:1773941541.997349:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997350:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997352:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997353:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 48 buffers still posted 00000100:00000001:3.0:1773941541.997357:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997358:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997359:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997360:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abb00. 00000400:00000200:3.0:1773941541.997361:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b98124780 00000400:00000010:3.0:1773941541.997362:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b98124780. 00000100:00000001:3.0:1773941541.997363:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997364:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997367:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997368:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 47 buffers still posted 00000100:00000001:3.0:1773941541.997369:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997370:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997370:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997371:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab630. 00000400:00000200:3.0:1773941541.997372:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b98124cc0 00000400:00000010:3.0:1773941541.997373:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b98124cc0. 00000100:00000001:3.0:1773941541.997374:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997375:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997377:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997378:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 46 buffers still posted 00000100:00000001:3.0:1773941541.997379:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997380:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997381:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997381:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab108. 00000400:00000200:3.0:1773941541.997382:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b98124300 00000400:00000010:3.0:1773941541.997384:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b98124300. 00000100:00000001:3.0:1773941541.997385:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997385:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997388:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997389:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 45 buffers still posted 00000100:00000001:3.0:1773941541.997390:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997391:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997391:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997392:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab790. 00000400:00000200:3.0:1773941541.997393:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b300 00000400:00000010:3.0:1773941541.997394:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b300. 00000100:00000001:3.0:1773941541.997397:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997398:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997403:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997404:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 44 buffers still posted 00000100:00000001:3.0:1773941541.997405:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997406:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997407:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997408:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab738. 00000400:00000200:3.0:1773941541.997409:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680ba80 00000400:00000010:3.0:1773941541.997410:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680ba80. 00000100:00000001:3.0:1773941541.997411:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997412:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997414:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997415:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 43 buffers still posted 00000100:00000001:3.0:1773941541.997416:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997417:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997418:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997419:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abcb8. 00000400:00000200:3.0:1773941541.997420:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b240 00000400:00000010:3.0:1773941541.997421:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b240. 00000100:00000001:3.0:1773941541.997422:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997423:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997425:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997426:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 42 buffers still posted 00000100:00000001:3.0:1773941541.997427:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997428:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997429:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997430:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abbb0. 00000400:00000200:3.0:1773941541.997431:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b900 00000400:00000010:3.0:1773941541.997432:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b900. 00000100:00000001:3.0:1773941541.997433:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997434:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997436:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997437:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 41 buffers still posted 00000100:00000001:3.0:1773941541.997438:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997440:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997440:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997441:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab268. 00000400:00000200:3.0:1773941541.997442:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b9c0 00000400:00000010:3.0:1773941541.997443:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b9c0. 00000100:00000001:3.0:1773941541.997447:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997448:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997452:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997453:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 40 buffers still posted 00000100:00000001:3.0:1773941541.997454:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997456:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997456:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997457:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab370. 00000400:00000200:3.0:1773941541.997458:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680bc00 00000400:00000010:3.0:1773941541.997459:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680bc00. 00000100:00000001:3.0:1773941541.997460:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997461:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997464:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997465:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 39 buffers still posted 00000100:00000001:3.0:1773941541.997465:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997466:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997467:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997468:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abe70. 00000400:00000200:3.0:1773941541.997469:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680bf00 00000400:00000010:3.0:1773941541.997470:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680bf00. 00000100:00000001:3.0:1773941541.997471:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997471:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997474:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997475:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 38 buffers still posted 00000100:00000001:3.0:1773941541.997476:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997477:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997477:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997478:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab898. 00000400:00000200:3.0:1773941541.997479:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680bcc0 00000400:00000010:3.0:1773941541.997480:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680bcc0. 00000100:00000001:3.0:1773941541.997481:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997482:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997485:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997486:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 37 buffers still posted 00000100:00000001:3.0:1773941541.997487:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997488:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997488:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997489:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab580. 00000400:00000200:3.0:1773941541.997493:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b3c0 00000400:00000010:3.0:1773941541.997494:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b3c0. 00000100:00000001:3.0:1773941541.997496:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997497:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997500:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997501:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 36 buffers still posted 00000100:00000001:3.0:1773941541.997502:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997503:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997504:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997505:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abf20. 00000400:00000200:3.0:1773941541.997506:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680be40 00000400:00000010:3.0:1773941541.997507:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680be40. 00000100:00000001:3.0:1773941541.997508:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997509:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997511:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997512:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 35 buffers still posted 00000100:00000001:3.0:1773941541.997513:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997514:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997515:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997516:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845abb58. 00000400:00000200:3.0:1773941541.997517:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b0c0 00000400:00000010:3.0:1773941541.997518:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b0c0. 00000100:00000001:3.0:1773941541.997519:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997519:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997522:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997523:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 34 buffers still posted 00000100:00000001:3.0:1773941541.997524:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997525:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997525:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997526:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab5d8. 00000400:00000200:3.0:1773941541.997527:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b180 00000400:00000010:3.0:1773941541.997528:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b180. 00000100:00000001:3.0:1773941541.997529:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997530:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997533:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997533:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 33 buffers still posted 00000100:00000001:3.0:1773941541.997534:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997539:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997540:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997541:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab000. 00000400:00000200:3.0:1773941541.997542:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b780 00000400:00000010:3.0:1773941541.997543:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b780. 00000100:00000001:3.0:1773941541.997544:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997545:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997547:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997548:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 32 buffers still posted 00000100:00000001:3.0:1773941541.997549:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997550:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997551:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997551:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab8f0. 00000400:00000200:3.0:1773941541.997553:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680bb40 00000400:00000010:3.0:1773941541.997554:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680bb40. 00000100:00000001:3.0:1773941541.997555:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997555:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997558:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997559:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 31 buffers still posted 00000100:00000001:3.0:1773941541.997560:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997561:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997561:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997562:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b845ab840. 00000400:00000200:3.0:1773941541.997563:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b8680b480 00000400:00000010:3.0:1773941541.997564:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b8680b480. 00000100:00000001:3.0:1773941541.997565:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997566:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997569:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997570:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 30 buffers still posted 00000100:00000001:3.0:1773941541.997571:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997571:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997572:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997573:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f872c0. 00000400:00000200:3.0:1773941541.997574:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7a80 00000400:00000010:3.0:1773941541.997575:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7a80. 00000100:00000001:3.0:1773941541.997576:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997577:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997580:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997584:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 29 buffers still posted 00000100:00000001:3.0:1773941541.997585:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997587:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997587:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997589:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f870b0. 00000400:00000200:3.0:1773941541.997590:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed73c0 00000400:00000010:3.0:1773941541.997591:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed73c0. 00000100:00000001:3.0:1773941541.997592:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997593:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997596:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997597:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 28 buffers still posted 00000100:00000001:3.0:1773941541.997598:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997599:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997600:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997601:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f874d0. 00000400:00000200:3.0:1773941541.997602:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7600 00000400:00000010:3.0:1773941541.997603:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7600. 00000100:00000001:3.0:1773941541.997604:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997605:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997607:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997608:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 27 buffers still posted 00000100:00000001:3.0:1773941541.997609:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997610:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997611:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997612:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87688. 00000400:00000200:3.0:1773941541.997613:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7cc0 00000400:00000010:3.0:1773941541.997613:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7cc0. 00000100:00000001:3.0:1773941541.997615:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997615:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997618:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997619:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 26 buffers still posted 00000100:00000001:3.0:1773941541.997620:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997621:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997621:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997622:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f876e0. 00000400:00000200:3.0:1773941541.997623:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed76c0 00000400:00000010:3.0:1773941541.997624:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed76c0. 00000100:00000001:3.0:1773941541.997625:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997629:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997632:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997633:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 25 buffers still posted 00000100:00000001:3.0:1773941541.997634:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997636:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997637:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997638:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87058. 00000400:00000200:3.0:1773941541.997639:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7900 00000400:00000010:3.0:1773941541.997640:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7900. 00000100:00000001:3.0:1773941541.997641:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997642:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997655:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997656:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 24 buffers still posted 00000100:00000001:3.0:1773941541.997657:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997658:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997658:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997659:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87aa8. 00000400:00000200:3.0:1773941541.997660:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7f00 00000400:00000010:3.0:1773941541.997661:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7f00. 00000100:00000001:3.0:1773941541.997662:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997663:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997666:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997667:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 23 buffers still posted 00000100:00000001:3.0:1773941541.997668:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997669:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997669:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997670:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87210. 00000400:00000200:3.0:1773941541.997671:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7e40 00000400:00000010:3.0:1773941541.997672:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7e40. 00000100:00000001:3.0:1773941541.997673:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997674:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997677:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997678:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 22 buffers still posted 00000100:00000001:3.0:1773941541.997678:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997679:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997680:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997681:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87c60. 00000400:00000200:3.0:1773941541.997682:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed79c0 00000400:00000010:3.0:1773941541.997686:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed79c0. 00000100:00000001:3.0:1773941541.997687:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997688:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997691:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997692:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 21 buffers still posted 00000100:00000001:3.0:1773941541.997692:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997694:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997694:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997695:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87108. 00000400:00000200:3.0:1773941541.997696:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7000 00000400:00000010:3.0:1773941541.997697:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7000. 00000100:00000001:3.0:1773941541.997698:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997699:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997720:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997722:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 20 buffers still posted 00000100:00000001:3.0:1773941541.997724:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997725:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997726:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997727:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87790. 00000400:00000200:3.0:1773941541.997730:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7840 00000400:00000010:3.0:1773941541.997731:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7840. 00000100:00000001:3.0:1773941541.997734:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997735:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997739:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997741:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 19 buffers still posted 00000100:00000001:3.0:1773941541.997742:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997744:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997746:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997747:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87738. 00000400:00000200:3.0:1773941541.997749:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7b40 00000400:00000010:3.0:1773941541.997750:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7b40. 00000100:00000001:3.0:1773941541.997751:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997752:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997755:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997756:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 18 buffers still posted 00000100:00000001:3.0:1773941541.997756:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997758:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997762:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997763:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87cb8. 00000400:00000200:3.0:1773941541.997765:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7240 00000400:00000010:3.0:1773941541.997766:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7240. 00000100:00000001:3.0:1773941541.997768:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997769:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997772:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997773:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 17 buffers still posted 00000100:00000001:3.0:1773941541.997774:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997775:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997776:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997777:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87bb0. 00000400:00000200:3.0:1773941541.997778:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7300 00000400:00000010:3.0:1773941541.997779:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7300. 00000100:00000001:3.0:1773941541.997780:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997780:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997783:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997784:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 16 buffers still posted 00000100:00000001:3.0:1773941541.997785:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997786:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997787:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997787:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87268. 00000400:00000200:3.0:1773941541.997788:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7c00 00000400:00000010:3.0:1773941541.997789:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7c00. 00000100:00000001:3.0:1773941541.997790:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997791:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997794:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997795:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 15 buffers still posted 00000100:00000001:3.0:1773941541.997796:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997797:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997797:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997798:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87370. 00000400:00000200:3.0:1773941541.997799:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7180 00000400:00000010:3.0:1773941541.997800:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7180. 00000100:00000001:3.0:1773941541.997801:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997802:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997805:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997806:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 14 buffers still posted 00000100:00000001:3.0:1773941541.997809:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997811:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997812:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997813:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87e70. 00000400:00000200:3.0:1773941541.997815:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7d80 00000400:00000010:3.0:1773941541.997816:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7d80. 00000100:00000001:3.0:1773941541.997817:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997818:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997821:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997822:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 13 buffers still posted 00000100:00000001:3.0:1773941541.997823:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997824:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997824:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997825:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87898. 00000400:00000200:3.0:1773941541.997826:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7780 00000400:00000010:3.0:1773941541.997827:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7780. 00000100:00000001:3.0:1773941541.997828:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997829:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997832:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997833:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 12 buffers still posted 00000100:00000001:3.0:1773941541.997834:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997835:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997835:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997836:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87580. 00000400:00000200:3.0:1773941541.997837:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7540 00000400:00000010:3.0:1773941541.997838:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7540. 00000100:00000001:3.0:1773941541.997840:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997840:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997843:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997844:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 11 buffers still posted 00000100:00000001:3.0:1773941541.997845:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997856:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997857:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997858:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87f20. 00000400:00000200:3.0:1773941541.997859:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed7480 00000400:00000010:3.0:1773941541.997860:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed7480. 00000100:00000001:3.0:1773941541.997861:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997862:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997867:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997869:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 10 buffers still posted 00000100:00000001:3.0:1773941541.997870:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997872:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997873:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997874:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87b58. 00000400:00000200:3.0:1773941541.997875:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b85ed70c0 00000400:00000010:3.0:1773941541.997876:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b85ed70c0. 00000100:00000001:3.0:1773941541.997878:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997879:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997884:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997885:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 9 buffers still posted 00000100:00000001:3.0:1773941541.997887:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997889:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997891:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997892:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f875d8. 00000400:00000200:3.0:1773941541.997895:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b88ca6cc0 00000400:00000010:3.0:1773941541.997897:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b88ca6cc0. 00000100:00000001:3.0:1773941541.997900:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997902:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997907:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997909:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 8 buffers still posted 00000100:00000001:3.0:1773941541.997910:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997923:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997925:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997926:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87000. 00000400:00000200:3.0:1773941541.997929:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b88ca6d80 00000400:00000010:3.0:1773941541.997931:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b88ca6d80. 00000100:00000001:3.0:1773941541.997934:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997935:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997940:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997942:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 7 buffers still posted 00000100:00000001:3.0:1773941541.997943:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997945:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997946:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997947:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f878f0. 00000400:00000200:3.0:1773941541.997949:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b91fc0780 00000400:00000010:3.0:1773941541.997950:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b91fc0780. 00000100:00000001:3.0:1773941541.997964:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997965:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997969:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997981:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 6 buffers still posted 00000100:00000001:3.0:1773941541.997982:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997984:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.997985:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.997986:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87840. 00000400:00000200:3.0:1773941541.997988:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b91fc0e40 00000400:00000010:3.0:1773941541.997989:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b91fc0e40. 00000100:00000001:3.0:1773941541.997991:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.997992:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.997995:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.997997:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 5 buffers still posted 00000100:00000001:3.0:1773941541.997998:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.997999:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.998000:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.998001:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87dc0. 00000400:00000200:3.0:1773941541.998002:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b91fc0900 00000400:00000010:3.0:1773941541.998003:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b91fc0900. 00000100:00000001:3.0:1773941541.998004:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.998005:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.998008:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.998009:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 4 buffers still posted 00000100:00000001:3.0:1773941541.998010:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.998022:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.998023:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.998024:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f87ec8. 00000400:00000200:3.0:1773941541.998025:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b91fc0cc0 00000400:00000010:3.0:1773941541.998026:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b91fc0cc0. 00000100:00000001:3.0:1773941541.998027:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.998028:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.998031:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.998032:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 3 buffers still posted 00000100:00000001:3.0:1773941541.998032:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.998033:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.998034:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.998035:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f877e8. 00000400:00000200:3.0:1773941541.998040:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90ec5480 00000400:00000010:3.0:1773941541.998041:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90ec5480. 00000100:00000001:3.0:1773941541.998042:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.998043:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.998046:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.998047:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 2 buffers still posted 00000100:00000001:3.0:1773941541.998048:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.998049:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.998050:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.998051:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f879a0. 00000400:00000200:3.0:1773941541.998052:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90ec5b40 00000400:00000010:3.0:1773941541.998053:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90ec5b40. 00000100:00000001:3.0:1773941541.998054:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.998054:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.998057:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.998058:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 1 buffers still posted 00000100:00000001:3.0:1773941541.998059:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941541.998060:0:12130:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941541.998061:0:12130:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941541.998061:0:12130:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b5b85f873c8. 00000400:00000200:3.0:1773941541.998063:0:12130:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b5b90ec5000 00000400:00000010:3.0:1773941541.998063:0:12130:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff8b5b90ec5000. 00000100:00000001:3.0:1773941541.998064:0:12130:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941541.998065:0:12130:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941541.998068:0:12130:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941541.998069:0:12130:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:3.0:1773941541.998069:0:12130:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1773941541.998071:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998072:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998073:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998074:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998074:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998075:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998076:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998077:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998077:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998078:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998079:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998089:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998090:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998091:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998092:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998093:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998094:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998095:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998096:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998096:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998097:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998098:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998099:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998099:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998100:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998101:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998102:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998102:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998103:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998104:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998104:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998105:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998106:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998107:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998108:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998108:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998109:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998110:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998111:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998111:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998112:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998113:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998114:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998114:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998115:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998116:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998117:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998117:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998118:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998119:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998119:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998123:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998124:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998125:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998126:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998126:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998128:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998129:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998129:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998130:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998131:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998131:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998132:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998133:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998134:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998134:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998135:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998136:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998137:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998137:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998138:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998139:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998140:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998140:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998141:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998142:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998143:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998143:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998144:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998145:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998145:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998146:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998147:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998148:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998148:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998149:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998150:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998151:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998151:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998152:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998153:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998156:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998157:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998158:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998158:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998159:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998160:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998161:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998161:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998162:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998163:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998163:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998164:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998165:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998166:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998166:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998167:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998168:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998169:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998169:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998170:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998171:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998172:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998172:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998173:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998174:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998175:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998175:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998176:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998177:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998178:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998178:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998179:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998180:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998180:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998181:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998182:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998183:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:3.0:1773941541.998185:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b850d8000. 00000100:00000010:3.0:1773941541.998187:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b98f0f000. 00000100:00000010:3.0:1773941541.998189:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b850da000. 00000100:00000010:3.0:1773941541.998194:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b98f0dc00. 00000100:00000010:3.0:1773941541.998196:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b72000. 00000100:00000010:3.0:1773941541.998200:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b98f0fc00. 00000100:00000010:3.0:1773941541.998202:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b76000. 00000100:00000010:3.0:1773941541.998204:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b84c97400. 00000100:00000010:3.0:1773941541.998213:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b74000. 00000100:00000010:3.0:1773941541.998216:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b84c96000. 00000100:00000010:3.0:1773941541.998218:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b70000. 00000100:00000010:3.0:1773941541.998221:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b84c95800. 00000100:00000010:3.0:1773941541.998223:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b8856a000. 00000100:00000010:3.0:1773941541.998226:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b84c96400. 00000100:00000010:3.0:1773941541.998227:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b8856e000. 00000100:00000010:3.0:1773941541.998229:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b84c95000. 00000100:00000010:3.0:1773941541.998231:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b8856c000. 00000100:00000010:3.0:1773941541.998233:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b84c94400. 00000100:00000010:3.0:1773941541.998234:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88568000. 00000100:00000010:3.0:1773941541.998237:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b84c97800. 00000100:00000010:3.0:1773941541.998238:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88560000. 00000100:00000010:3.0:1773941541.998241:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b84c96800. 00000100:00000010:3.0:1773941541.998242:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88562000. 00000100:00000010:3.0:1773941541.998245:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b84c97000. 00000100:00000010:3.0:1773941541.998247:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88566000. 00000100:00000010:3.0:1773941541.998250:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b84c94c00. 00000100:00000010:3.0:1773941541.998251:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88564000. 00000100:00000010:3.0:1773941541.998253:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b84c96c00. 00000100:00000010:3.0:1773941541.998255:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885ce000. 00000100:00000010:3.0:1773941541.998257:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b84c95c00. 00000100:00000010:3.0:1773941541.998259:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885cc000. 00000100:00000010:3.0:1773941541.998261:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b84c95400. 00000100:00000010:3.0:1773941541.998262:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885c8000. 00000100:00000010:3.0:1773941541.998264:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b84c94800. 00000100:00000010:3.0:1773941541.998265:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885ca000. 00000100:00000010:3.0:1773941541.998267:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8902ac00. 00000100:00000010:3.0:1773941541.998272:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885c4000. 00000100:00000010:3.0:1773941541.998275:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0dc4800. 00000100:00000010:3.0:1773941541.998277:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885c0000. 00000100:00000010:3.0:1773941541.998279:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0dc5c00. 00000100:00000010:3.0:1773941541.998281:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885c2000. 00000100:00000010:3.0:1773941541.998283:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0dc5400. 00000100:00000010:3.0:1773941541.998284:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885c6000. 00000100:00000010:3.0:1773941541.998286:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0dc5800. 00000100:00000010:3.0:1773941541.998287:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b850b8000. 00000100:00000010:3.0:1773941541.998299:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88cb3000. 00000100:00000010:3.0:1773941541.998301:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b850ba000. 00000100:00000010:3.0:1773941541.998303:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88cb3800. 00000100:00000010:3.0:1773941541.998304:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b850be000. 00000100:00000010:3.0:1773941541.998316:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88cb1c00. 00000100:00000010:3.0:1773941541.998317:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b850bc000. 00000100:00000010:3.0:1773941541.998319:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8800f800. 00000100:00000010:3.0:1773941541.998322:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b981a6000. 00000100:00000010:3.0:1773941541.998325:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8800d400. 00000100:00000010:3.0:1773941541.998327:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b981a2000. 00000100:00000010:3.0:1773941541.998330:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8800e000. 00000100:00000010:3.0:1773941541.998332:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b981a0000. 00000100:00000010:3.0:1773941541.998335:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8800d800. 00000100:00000010:3.0:1773941541.998337:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b78000. 00000100:00000010:3.0:1773941541.998339:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8800c000. 00000100:00000010:3.0:1773941541.998340:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b7e000. 00000100:00000010:3.0:1773941541.998342:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8800f000. 00000100:00000010:3.0:1773941541.998343:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b7c000. 00000100:00000010:3.0:1773941541.998346:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8800dc00. 00000100:00000010:3.0:1773941541.998347:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b83d0a000. 00000100:00000010:3.0:1773941541.998354:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8800cc00. 00000100:00000010:3.0:1773941541.998355:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b829e4000. 00000100:00000010:3.0:1773941541.998358:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8800c400. 00000100:00000010:3.0:1773941541.998360:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b829e2000. 00000100:00000010:3.0:1773941541.998365:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8800fc00. 00000100:00000010:3.0:1773941541.998366:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b91ca8000. 00000100:00000010:3.0:1773941541.998370:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8800ec00. 00000100:00000010:3.0:1773941541.998372:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b91cac000. 00000100:00000010:3.0:1773941541.998374:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8800e400. 00000100:00000010:3.0:1773941541.998375:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b832aa000. 00000100:00000010:3.0:1773941541.998378:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0b92c00. 00000100:00000010:3.0:1773941541.998381:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b832a8000. 00000100:00000010:3.0:1773941541.998383:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0b91400. 00000100:00000010:3.0:1773941541.998385:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b832ac000. 00000100:00000010:3.0:1773941541.998387:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0b91000. 00000100:00000010:3.0:1773941541.998388:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b8857a000. 00000100:00000010:3.0:1773941541.998390:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0b93c00. 00000100:00000010:3.0:1773941541.998392:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88578000. 00000100:00000010:3.0:1773941541.998394:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0b92800. 00000100:00000010:3.0:1773941541.998396:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b8857c000. 00000100:00000010:3.0:1773941541.998398:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0b91800. 00000100:00000010:3.0:1773941541.998399:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b8857e000. 00000100:00000010:3.0:1773941541.998402:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0b90c00. 00000100:00000010:3.0:1773941541.998404:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90278000. 00000100:00000010:3.0:1773941541.998406:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0b90400. 00000100:00000010:3.0:1773941541.998407:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b9027c000. 00000100:00000010:3.0:1773941541.998409:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0b92000. 00000100:00000010:3.0:1773941541.998410:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b9027e000. 00000100:00000010:3.0:1773941541.998412:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0b93800. 00000100:00000010:3.0:1773941541.998414:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b9027a000. 00000100:00000010:3.0:1773941541.998416:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0b92400. 00000100:00000010:3.0:1773941541.998417:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889da000. 00000100:00000010:3.0:1773941541.998426:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0b91c00. 00000100:00000010:3.0:1773941541.998428:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889d8000. 00000100:00000010:3.0:1773941541.998430:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0b90000. 00000100:00000010:3.0:1773941541.998431:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889dc000. 00000100:00000010:3.0:1773941541.998433:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0b90800. 00000100:00000010:3.0:1773941541.998438:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889de000. 00000100:00000010:3.0:1773941541.998440:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91f8ec00. 00000100:00000010:3.0:1773941541.998443:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88988000. 00000100:00000010:3.0:1773941541.998446:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91f8d400. 00000100:00000010:3.0:1773941541.998448:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b8898c000. 00000100:00000010:3.0:1773941541.998450:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91f8cc00. 00000100:00000010:3.0:1773941541.998451:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b8898e000. 00000100:00000010:3.0:1773941541.998453:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91f8e800. 00000100:00000010:3.0:1773941541.998454:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b8898a000. 00000100:00000010:3.0:1773941541.998457:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91f8f400. 00000100:00000010:3.0:1773941541.998458:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0cd6000. 00000100:00000010:3.0:1773941541.998461:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91f8e400. 00000100:00000010:3.0:1773941541.998462:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0cd2000. 00000100:00000010:3.0:1773941541.998466:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88011800. 00000100:00000010:3.0:1773941541.998468:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0cd0000. 00000100:00000010:3.0:1773941541.998470:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88012400. 00000100:00000010:3.0:1773941541.998472:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0cd4000. 00000100:00000010:3.0:1773941541.998474:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88010800. 00000100:00000010:3.0:1773941541.998476:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0c00000. 00000100:00000010:3.0:1773941541.998487:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88011000. 00000100:00000010:3.0:1773941541.998489:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0c04000. 00000100:00000010:3.0:1773941541.998492:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88012800. 00000100:00000010:3.0:1773941541.998494:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0c06000. 00000100:00000010:3.0:1773941541.998496:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88011c00. 00000100:00000010:3.0:1773941541.998498:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0c02000. 00000100:00000010:3.0:1773941541.998500:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b88012000. 00000100:00000010:3.0:1773941541.998503:0:12130:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b885e5800. 00000100:00000010:3.0:1773941541.998504:0:12130:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b8896a000. 00000100:00000010:3.0:1773941541.998506:0:12130:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b8896a800. 00000100:00000001:3.0:1773941541.998508:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998509:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998510:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998511:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998515:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998516:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998517:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998518:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998519:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998520:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998521:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998521:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998522:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998523:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998524:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998524:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998525:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998526:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998527:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998527:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998528:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998529:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998530:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998530:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998531:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998532:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998533:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998533:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998534:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998535:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998535:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998536:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998537:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998538:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998538:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998539:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998540:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998541:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998541:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998542:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998543:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998543:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998544:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998545:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998548:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998549:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998550:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998551:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998552:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998553:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998554:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998555:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998556:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998556:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998557:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998558:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998559:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998559:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998560:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998561:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998562:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998562:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998563:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998564:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998564:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998565:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998566:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998567:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998567:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998568:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998569:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998570:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998570:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998571:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998572:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998573:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998573:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998574:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998575:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998575:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998576:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998577:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998578:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998578:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998582:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998583:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998584:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998585:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998586:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998587:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998588:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998588:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998589:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998590:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998591:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998591:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998592:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998593:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998594:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998594:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998595:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998596:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998597:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998597:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998598:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998599:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998600:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998600:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998601:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998602:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998603:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998603:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998604:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998605:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998606:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998606:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998607:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998608:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998608:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998609:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998610:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998611:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998611:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998612:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998616:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998617:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941541.998618:0:12130:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941541.998619:0:12130:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:3.0:1773941541.998621:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90df0000. 00000100:00000010:3.0:1773941541.998623:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91c4d400. 00000100:00000010:3.0:1773941541.998625:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90df4000. 00000100:00000010:3.0:1773941541.998627:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91c4d000. 00000100:00000010:3.0:1773941541.998629:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90df6000. 00000100:00000010:3.0:1773941541.998631:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91c4fc00. 00000100:00000010:3.0:1773941541.998633:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90df2000. 00000100:00000010:3.0:1773941541.998636:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91e4a400. 00000100:00000010:3.0:1773941541.998638:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d4e000. 00000100:00000010:3.0:1773941541.998640:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b91e49000. 00000100:00000010:3.0:1773941541.998641:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d4a000. 00000100:00000010:3.0:1773941541.998644:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b890c3000. 00000100:00000010:3.0:1773941541.998646:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d48000. 00000100:00000010:3.0:1773941541.998664:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b890c0800. 00000100:00000010:3.0:1773941541.998667:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d4c000. 00000100:00000010:3.0:1773941541.998671:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b890c2000. 00000100:00000010:3.0:1773941541.998675:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889ca000. 00000100:00000010:3.0:1773941541.998697:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b99ef4000. 00000100:00000010:3.0:1773941541.998701:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889c8000. 00000100:00000010:3.0:1773941541.998704:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b99ef7400. 00000100:00000010:3.0:1773941541.998707:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889cc000. 00000100:00000010:3.0:1773941541.998711:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b99ef4800. 00000100:00000010:3.0:1773941541.998729:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889ce000. 00000100:00000010:3.0:1773941541.998768:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f3a400. 00000100:00000010:3.0:1773941541.998774:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90256000. 00000100:00000010:3.0:1773941541.998796:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f3a800. 00000100:00000010:3.0:1773941541.998798:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90252000. 00000100:00000010:3.0:1773941541.998802:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f3ac00. 00000100:00000010:3.0:1773941541.998804:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90250000. 00000100:00000010:3.0:1773941541.998807:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f38400. 00000100:00000010:3.0:1773941541.998815:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90254000. 00000100:00000010:3.0:1773941541.998817:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f38800. 00000100:00000010:3.0:1773941541.998819:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90242000. 00000100:00000010:3.0:1773941541.998822:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f3bc00. 00000100:00000010:3.0:1773941541.998824:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90240000. 00000100:00000010:3.0:1773941541.998827:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f39800. 00000100:00000010:3.0:1773941541.998829:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90244000. 00000100:00000010:3.0:1773941541.998831:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f39000. 00000100:00000010:3.0:1773941541.998833:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90246000. 00000100:00000010:3.0:1773941541.998836:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b90f3b800. 00000100:00000010:3.0:1773941541.998838:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885f4000. 00000100:00000010:3.0:1773941541.998867:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b99f56800. 00000100:00000010:3.0:1773941541.998869:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885f6000. 00000100:00000010:3.0:1773941541.998872:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b99f57800. 00000100:00000010:3.0:1773941541.998874:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885f2000. 00000100:00000010:3.0:1773941541.998876:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b99f54400. 00000100:00000010:3.0:1773941541.998878:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b885f0000. 00000100:00000010:3.0:1773941541.998880:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b99f55000. 00000100:00000010:3.0:1773941541.998881:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b9022e000. 00000100:00000010:3.0:1773941541.998884:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5bc2072000. 00000100:00000010:3.0:1773941541.998886:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b9022a000. 00000100:00000010:3.0:1773941541.998888:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5bc2073400. 00000100:00000010:3.0:1773941541.998890:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90228000. 00000100:00000010:3.0:1773941541.998892:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8a3a2c00. 00000100:00000010:3.0:1773941541.998894:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b9022c000. 00000100:00000010:3.0:1773941541.998896:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5bc2076000. 00000100:00000010:3.0:1773941541.998898:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b8505a000. 00000100:00000010:3.0:1773941541.998901:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5bc2068000. 00000100:00000010:3.0:1773941541.998903:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b85058000. 00000100:00000010:3.0:1773941541.998904:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5bc206b000. 00000100:00000010:3.0:1773941541.998906:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b8505c000. 00000100:00000010:3.0:1773941541.998908:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5bc2069c00. 00000100:00000010:3.0:1773941541.998909:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b8505e000. 00000100:00000010:3.0:1773941541.998915:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5bc2069400. 00000100:00000010:3.0:1773941541.998916:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889f0000. 00000100:00000010:3.0:1773941541.998928:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85611000. 00000100:00000010:3.0:1773941541.998930:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889f4000. 00000100:00000010:3.0:1773941541.998933:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85610c00. 00000100:00000010:3.0:1773941541.998935:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889f6000. 00000100:00000010:3.0:1773941541.998937:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b85613800. 00000100:00000010:3.0:1773941541.998939:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b889f2000. 00000100:00000010:3.0:1773941541.998941:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b8801c800. 00000100:00000010:3.0:1773941541.998943:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88992000. 00000100:00000010:3.0:1773941541.998945:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0bacc00. 00000100:00000010:3.0:1773941541.998947:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88990000. 00000100:00000010:3.0:1773941541.998949:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0baec00. 00000100:00000010:3.0:1773941541.998951:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88994000. 00000100:00000010:3.0:1773941541.998953:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0badc00. 00000100:00000010:3.0:1773941541.998954:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88996000. 00000100:00000010:3.0:1773941541.998956:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0bad400. 00000100:00000010:3.0:1773941541.998958:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a3e000. 00000100:00000010:3.0:1773941541.998960:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0bafc00. 00000100:00000010:3.0:1773941541.998963:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a3a000. 00000100:00000010:3.0:1773941541.998965:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0bac800. 00000100:00000010:3.0:1773941541.998967:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a38000. 00000100:00000010:3.0:1773941541.998969:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0bad800. 00000100:00000010:3.0:1773941541.998982:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a3c000. 00000100:00000010:3.0:1773941541.998984:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0baf800. 00000100:00000010:3.0:1773941541.998986:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d6a000. 00000100:00000010:3.0:1773941541.998998:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0bac400. 00000100:00000010:3.0:1773941541.999000:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d68000. 00000100:00000010:3.0:1773941541.999002:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0bad000. 00000100:00000010:3.0:1773941541.999004:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d6c000. 00000100:00000010:3.0:1773941541.999006:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0bae400. 00000100:00000010:3.0:1773941541.999007:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90d6e000. 00000100:00000010:3.0:1773941541.999009:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0bae800. 00000100:00000010:3.0:1773941541.999014:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a12000. 00000100:00000010:3.0:1773941541.999017:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0bae000. 00000100:00000010:3.0:1773941541.999018:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a10000. 00000100:00000010:3.0:1773941541.999020:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0baf400. 00000100:00000010:3.0:1773941541.999022:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a14000. 00000100:00000010:3.0:1773941541.999024:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0bac000. 00000100:00000010:3.0:1773941541.999025:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b98a16000. 00000100:00000010:3.0:1773941541.999027:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5ba0baf000. 00000100:00000010:3.0:1773941541.999028:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b18000. 00000100:00000010:3.0:1773941541.999031:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b850c2c00. 00000100:00000010:3.0:1773941541.999033:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b1c000. 00000100:00000010:3.0:1773941541.999035:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b850c1c00. 00000100:00000010:3.0:1773941541.999037:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b1e000. 00000100:00000010:3.0:1773941541.999039:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b850c1400. 00000100:00000010:3.0:1773941541.999040:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5ba0b1a000. 00000100:00000010:3.0:1773941541.999043:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b850c3c00. 00000100:00000010:3.0:1773941541.999044:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90da4000. 00000100:00000010:3.0:1773941541.999056:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b850c0800. 00000100:00000010:3.0:1773941541.999058:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90da6000. 00000100:00000010:3.0:1773941541.999061:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b850c1800. 00000100:00000010:3.0:1773941541.999062:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90da2000. 00000100:00000010:3.0:1773941541.999065:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b850c3800. 00000100:00000010:3.0:1773941541.999066:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b90da0000. 00000100:00000010:3.0:1773941541.999068:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b850c0400. 00000100:00000010:3.0:1773941541.999070:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88944000. 00000100:00000010:3.0:1773941541.999072:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b850c1000. 00000100:00000010:3.0:1773941541.999073:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88946000. 00000100:00000010:3.0:1773941541.999075:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b850c2400. 00000100:00000010:3.0:1773941541.999077:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88942000. 00000100:00000010:3.0:1773941541.999079:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b850c2800. 00000100:00000010:3.0:1773941541.999080:0:12130:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff8b5b88940000. 00000100:00000010:3.0:1773941541.999082:0:12130:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff8b5b850c2000. 00000100:00000010:3.0:1773941541.999083:0:12130:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b8907b000. 00000100:00000010:3.0:1773941541.999089:0:12130:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b90f20800. 00000100:00000010:3.0:1773941541.999090:0:12130:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff8b5b98ac1000. 00000100:00000001:3.0:1773941541.999093:0:12130:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:3.0:1773941541.999095:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999096:0:12130:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941541.999097:0:12130:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941541.999098:0:12130:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff8b5ba039c1c0. 00000100:00000001:3.0:1773941541.999101:0:12130:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941541.999102:0:12130:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.999103:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999104:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b88ea7480. 00000100:00000001:3.0:1773941541.999106:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999107:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999108:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b88ea7a80. 00000100:00000001:3.0:1773941541.999109:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999110:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999111:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b88ea7f00. 00000100:00000001:3.0:1773941541.999112:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999113:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999113:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b88ea7780. 00000100:00000001:3.0:1773941541.999115:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999116:0:12130:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941541.999116:0:12130:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941541.999117:0:12130:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff8b5ba039c100. 00000100:00000001:3.0:1773941541.999119:0:12130:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941541.999119:0:12130:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.999120:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999121:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b88ea7900. 00000100:00000001:3.0:1773941541.999122:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999123:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999124:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b88ea76c0. 00000100:00000001:3.0:1773941541.999125:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999126:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999126:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b88ea7b40. 00000100:00000001:3.0:1773941541.999127:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999128:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999129:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b88ea7e40. 00000100:00000010:3.0:1773941541.999130:0:12130:0:(nrs.c:1080:nrs_svcpt_cleanup_locked()) kfreed 'nrs': 112 at ffff8b5b8848fc80. 00000100:00000001:3.0:1773941541.999132:0:12130:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000100:00000001:3.0:1773941541.999135:0:12130:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:3.0:1773941541.999136:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999137:0:12130:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941541.999138:0:12130:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941541.999140:0:12130:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff8b5ba039ce80. 00000100:00000001:3.0:1773941541.999141:0:12130:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941541.999142:0:12130:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.999143:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999144:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b88ea7180. 00000100:00000001:3.0:1773941541.999145:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999146:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999147:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b88ea73c0. 00000100:00000001:3.0:1773941541.999148:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999149:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999149:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b88ea7000. 00000100:00000001:3.0:1773941541.999151:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999151:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999152:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b88ea7840. 00000100:00000001:3.0:1773941541.999154:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999154:0:12130:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941541.999155:0:12130:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941541.999156:0:12130:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff8b5ba039c3c0. 00000100:00000001:3.0:1773941541.999157:0:12130:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941541.999158:0:12130:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.999159:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999160:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b87df9900. 00000100:00000001:3.0:1773941541.999162:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999163:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999164:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b89064780. 00000100:00000001:3.0:1773941541.999165:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999166:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999166:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b89064c00. 00000100:00000001:3.0:1773941541.999168:0:12130:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941541.999168:0:12130:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941541.999169:0:12130:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff8b5b89064a80. 00000100:00000010:3.0:1773941541.999170:0:12130:0:(nrs.c:1080:nrs_svcpt_cleanup_locked()) kfreed 'nrs': 112 at ffff8b5b8848f000. 00000100:00000001:3.0:1773941541.999171:0:12130:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000020:00000010:3.0:1773941541.999222:0:12130:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 3680 at ffff8b5b885bb000. 00000020:00000010:3.0:1773941541.999226:0:12130:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff8b5b850c3400. 00000100:00000010:3.0:1773941541.999241:0:12130:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff8b5ba0c4c000. 00000100:00000010:3.0:1773941541.999251:0:12130:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff8b5b885bd000. 00000100:00000010:3.0:1773941541.999255:0:12130:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff8b5ba0718000. 00000100:00000010:3.0:1773941541.999260:0:12130:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff8b5b885b9000. 00000100:00000010:3.0:1773941541.999262:0:12130:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff8b5b98f0c800. 00000100:00000010:3.0:1773941541.999264:0:12130:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff8b5b88011400. 00000100:00000010:3.0:1773941541.999266:0:12130:0:(service.c:3836:ptlrpc_service_free()) kfreed 'svc': 360 at ffff8b5b845c4200. 00000100:00000001:3.0:1773941541.999267:0:12130:0:(service.c:3863:ptlrpc_unregister_service()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941541.999366:0:12143:0:(ldlm_lockd.c:142:have_expired_locks()) Process entered 00010000:00000001:1.0:1773941541.999369:0:12143:0:(ldlm_lockd.c:147:have_expired_locks()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941541.999381:0:12143:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff8b5b85068d40. 00000020:00000010:1.0:1773941541.999389:0:12143:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5b91ecb800. 00010000:00000010:3.0:1773941541.999392:0:12130:0:(ldlm_lockd.c:3586:ldlm_cleanup()) kfreed 'ldlm_state': 32 at ffff8b5b83f0c2e0. 00000040:00000010:1.0:1773941541.999394:0:12143:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5b91eca600. 00010000:00000001:3.0:1773941541.999398:0:12130:0:(ldlm_lockd.c:3589:ldlm_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941541.999400:0:12143:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d96800. 00010000:00000001:3.0:1773941541.999402:0:12130:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00000020:00000010:1.0:1773941541.999404:0:12143:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff8b5b8849cb78. 00010000:00000010:3.0:1773941541.999405:0:12130:0:(ldlm_lib.c:647:client_obd_cleanup()) kfreed 'cli->cl_read_io_latency_by_size': 3920 at ffff8b5b98acd000. 00000020:00000010:1.0:1773941541.999410:0:12143:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5b91ecbc00. 00010000:00000010:3.0:1773941541.999412:0:12130:0:(ldlm_lib.c:651:client_obd_cleanup()) kfreed 'cli->cl_write_io_latency_by_size': 3920 at ffff8b5b98acc000. 80000000:00000010:1.0:1773941541.999415:0:12143:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff8b5ba0bc92a0. 00010000:00000001:3.0:1773941541.999417:0:12130:0:(ldlm_lib.c:654:client_obd_cleanup()) Process leaving 40000000:00000010:1.0:1773941541.999419:0:12143:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff8b5b85068800. 10000000:00000010:3.0:1773941541.999420:0:12130:0:(mgc_request.c:834:mgc_device_free()) kfreed 'lu': 64 at ffff8b5ba039c580. 00100000:00000010:1.0:1773941541.999425:0:12143:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff8b5b90d92000. 10000000:00000001:3.0:1773941541.999426:0:12130:0:(mgc_request.c:836:mgc_device_free()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:1.0:1773941541.999429:0:12143:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98a69b00. 00000020:00000010:3.0:1773941541.999430:0:12130:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8b5ba0ab9000. 00040000:00000010:1.0:1773941541.999433:0:12143:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff8b5b90d96000. 00000040:00000010:3.0:1773941541.999435:0:12130:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff8b5ba0ab9800. 00040000:00000010:1.0:1773941541.999438:0:12143:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5b91ecae00. 00000020:00000010:3.0:1773941541.999440:0:12130:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff8b5b90d94000. 00000004:00000010:1.0:1773941541.999442:0:12143:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff8b5b90d91800. 00000020:00000010:3.0:1773941541.999445:0:12130:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff8b5ba0ab9e00. 00000004:00000010:1.0:1773941541.999447:0:12143:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff8b5ba0b8e000. 00040000:00000010:3.0:1773941541.999452:0:12130:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff8b5b98a69d00. 00000004:00000010:1.0:1773941541.999453:0:12143:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff8b5b90d95000. 00040000:00000010:3.0:1773941541.999460:0:12130:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff8b5ba0ab9400. 00002000:00000010:1.0:1773941541.999461:0:12143:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff8b5b90261c00. 00000004:00000010:1.0:1773941541.999465:0:12143:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff8b5b90263400. 00000020:00000010:3.0:1773941541.999468:0:12130:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5ba0ab8600. 00000020:00000010:1.0:1773941541.999470:0:12143:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8b5b91e54c00. 00000020:00000001:3.0:1773941541.999473:0:12130:0:(obd_class.h:647:obd_cleanup()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941541.999475:0:12143:0:(ldlm_lockd.c:290:expired_lock_main()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941541.999476:0:12130:0:(genops.c:66:obd_device_free()) slab-freed '(obd)': 6584 at ffff8b5b982fb370. 00000020:00000001:3.0:1773941541.999485:0:12130:0:(obd_config.c:923:class_detach()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.999487:0:12130:0:(obd_config.c:1655:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1773941541.999489:0:12130:0:(obd_config.c:2523:class_manual_cleanup()) kfreed 'lcfg': 72 at ffff8b5ba0bc9420. 00000020:00000001:3.0:1773941541.999492:0:12130:0:(obd_config.c:2524:class_manual_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.999494:0:12130:0:(obd_mount.c:620:lustre_stop_mgc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.999497:0:12130:0:(obd_mount.c:720:lustre_put_lsi()) Process entered 00000020:01000004:3.0:1773941541.999499:0:12130:0:(obd_mount.c:724:lustre_put_lsi()) put ffff8b5ba0b09800 1 00000020:00000001:3.0:1773941541.999501:0:12130:0:(obd_mount.c:679:lustre_free_lsi()) Process entered 00000020:01000004:3.0:1773941541.999503:0:12130:0:(obd_mount.c:682:lustre_free_lsi()) Freeing lsi ffff8b5ba0b0e000 00000020:00000010:3.0:1773941541.999505:0:12130:0:(obd_mount.c:691:lustre_free_lsi()) kfreed 'lsi': 1880 at ffff8b5ba0b0e000. 00000020:00000001:3.0:1773941541.999514:0:12130:0:(obd_mount.c:693:lustre_free_lsi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.999516:0:12130:0:(obd_mount.c:727:lustre_put_lsi()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1773941541.999518:0:12130:0:(obd_mount.c:929:lustre_common_put_super()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.999520:0:12130:0:(cl_object.c:885:cl_env_cache_purge()) Process entered 00000020:00000001:3.0:1773941541.999522:0:12130:0:(cl_object.c:903:cl_env_cache_purge()) Process leaving (rc=4294967295 : 4294967295 : ffffffff) 00000080:00000001:3.0:1773941541.999524:0:12130:0:(llite_lib.c:1639:ll_put_super()) Process leaving 00000080:00000001:3.0:1773941541.999526:0:12130:0:(llite_lib.c:1538:ll_fill_super()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000080:00020000:3.0:1773941541.999529:0:12130:0:(super25.c:186:lustre_fill_super()) llite: Unable to mount : rc = -107 00000020:00000010:2.0:1773941542.002122:0:12130:0:(obd_mount.c:631:lustre_put_lsm_free()) kfreed 'lmd->lmd_dev': 48 at ffff8b5ba039cb00. 00000020:00000010:2.0:1773941542.002126:0:12130:0:(obd_mount.c:632:lustre_put_lsm_free()) kfreed 'lmd->lmd_profile': 14 at ffff8b5b8859f3d0. 00000020:00000010:2.0:1773941542.002130:0:12130:0:(obd_mount.c:640:lustre_put_lsm_free()) kfreed 'lmd->lmd_mgsname': 19 at ffff8b5b884b46a0. 00000020:00000010:2.0:1773941542.002132:0:12130:0:(obd_mount.c:642:lustre_put_lsm_free()) kfreed 'lmd->lmd_params': 4096 at ffff8b5b88006000. 00000020:00000010:2.0:1773941542.002146:0:12130:0:(obd_mount.c:644:lustre_put_lsm_free()) kfreed 'lmd': 128 at ffff8b5b8848f700. 00000001:00000001:3.0:1773941542.557824:0:12387:0:(debug.c:644:libcfs_debug_mark_buffer()) ************************************************** 00000001:02000400:3.0:1773941542.557829:0:12387:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: conf-sanity test_161b: @@@@@@ FAIL: mount with hostname failed 00000001:00000001:3.0:1773941542.560076:0:12387:0:(debug.c:647:libcfs_debug_mark_buffer()) ************************************************** Debug log: 72490 lines, 72490 kept, 0 dropped, 0 bad.